diff --git a/.agents/skills/database-access/SKILL.md b/.agents/skills/database-access/SKILL.md new file mode 100644 index 00000000000..796dfafe4f6 --- /dev/null +++ b/.agents/skills/database-access/SKILL.md @@ -0,0 +1,52 @@ +--- +name: database-access +description: Add database queries, extract existing Prisma calls into repositories, or explicitly port queries to Rust and SQLx without expanding migration scope +--- + +# Database access + +## Adding new database access + +- Put Python database access behind an operation in `litellm/repositories` +- Give callers typed inputs and outputs, not Prisma clients, models, or query dictionaries +- Keep query execution and backend selection inside the repository +- Prisma is fine for the implementation, adding a query does not require a Rust port +- Native Rust callers use `litellm-rust/crates/db` directly + +## Asked to move existing access behind a repository + +- Identify the calls being moved and their transaction boundary +- Move the existing Prisma implementation into the relevant repository +- Replace those direct database calls with repository operations +- Preserve return values, errors, and transaction behavior, with focused regression tests +- Keep this an extraction, not a backend migration or unrelated cleanup + +## An equivalent Rust or SQLx query already exists + +- Leave it and the current backend selection unchanged unless integration is explicitly requested +- Its existence alone does not justify switching a Prisma caller to Rust +- Repository extraction can keep using Prisma even when the Rust query is available + +## Asked to port a Prisma query to Rust or SQLx + +- Identify the existing inputs, outputs, errors, and transaction boundary +- Check `litellm-rust/crates/db` for an equivalent query and reuse it if its behavior matches +- Otherwise follow `litellm-rust/crates/db/AGENTS.md` + - Put static SQL in `queries/` and compile-check it with SQLx + - Keep typed inputs and reusable row projections in `src/` + - Refresh SQLx query metadata after SQL changes +- Keep Python conversion in the Python bridge, not in `litellm-rust/crates/db` +- When caller integration is requested, expose the Rust operation through the bridge and call it from the repository + - A query-only port can land without switching callers +- Verify behavior against real PostgreSQL before switching callers + - Check relevant null handling, conflicts, ordering, errors, and transaction semantics +- Remove the old implementation only when no callers need it + +## Migration context and boundaries + +- Long-term direction is to centralize Python database access in repositories, then replace their Prisma implementations incrementally with Rust APIs +- Mixed Prisma and Rust implementations are expected during the transition, unrelated existing calls can stay as they are +- Both backends may have separate pools through PgBouncer, size them together and verify pooling-mode compatibility +- PgBouncer does not share transactions between Prisma and SQLx, keep each atomic unit of work on one backend and transaction +- Never retry a failed Rust write through Prisma when it may already have committed +- Prisma owns schema migrations today, eventual SQLx migration ownership and Prisma removal are separate planned steps, not part of a query port diff --git a/.github/scripts/auto_update_price_and_context_window_file.py b/.github/scripts/auto_update_price_and_context_window_file.py deleted file mode 100644 index 4c9a05bd4f0..00000000000 --- a/.github/scripts/auto_update_price_and_context_window_file.py +++ /dev/null @@ -1,319 +0,0 @@ -import asyncio -import aiohttp -import json -import math -from typing import Any - -# Asynchronously fetch data from a given URL -async def fetch_data(url): - try: - # Create an asynchronous session - async with aiohttp.ClientSession() as session: - # Send a GET request to the URL - async with session.get(url) as resp: - # Raise an error if the response status is not OK - resp.raise_for_status() - # Parse the response JSON - resp_json = await resp.json() - print("Fetch the data from URL.") - # Return the 'data' field from the JSON response - return resp_json['data'] - except Exception as e: - # Print an error message if fetching data fails - print("Error fetching data from URL:", e) - return None - - -FRIENDLI_API_URL = "https://api.friendli.ai/serverless/v1/models" -FRIENDLI_PROVIDER = "friendliai" - -INHERITABLE_BASE_KEYS = ( - "supports_pdf_input", - "supports_assistant_prefill", - "supports_adaptive_thinking", - "supports_output_config", -) - -REASONING_EFFORT_LEVEL_ORDER = ("none", "minimal", "low", "medium", "high", "xhigh", "max") - - -def _find_base_model_entry(base_model: str, local_data: dict) -> str | None: - if not base_model: - return None - bm_tail = base_model.split("/")[-1].lower() - if base_model in local_data: - return base_model - for key in local_data: - if key.startswith("sample_spec") or key == "fallback_generalizations": - continue - if key.split("/")[-1].lower() == bm_tail: - return key - return None - - -def _reasoning_effort_levels(reasoning_options: list) -> list: - offered = { - val - for opt in reasoning_options or [] - if opt.get("type") == "effort" - for val in opt.get("values", []) - } - return [level for level in REASONING_EFFORT_LEVEL_ORDER if level in offered] - - -def _valid_token_price(value: object) -> bool: - try: - price = float(value) # pyright: ignore[reportArgumentType] # non-numeric values are rejected via the except - except (TypeError, ValueError): - return False - return math.isfinite(price) and price >= 0 - - -def _has_valid_token_prices(pricing: dict | None) -> bool: - prices = pricing or {} - return _valid_token_price(prices.get("input")) and _valid_token_price(prices.get("output")) - - -def _pricing(pricing: dict) -> dict: - out: dict[str, Any] = {} - if not pricing: - return out - if "input" in pricing: - out["input_cost_per_token"] = float(pricing["input"]) - if "output" in pricing: - out["output_cost_per_token"] = float(pricing["output"]) - if "input_cache_read" in pricing and pricing["input_cache_read"] is not None: - out["cache_read_input_token_cost"] = float(pricing["input_cache_read"]) - return out - - -def _modality_flags(input_mods: list) -> dict: - mods = input_mods or [] - has_image = "image" in mods - return { - "supports_vision": has_image, - "supports_image_input": has_image, - "supports_video_input": "video" in mods, - } - - -def transform_friendli_data(data: list, local_data: dict) -> dict: - transformed: dict[str, dict] = {} - if not data: - return transformed - for model in data: - # An unpriced row must never wholesale-replace an already priced local entry: - # missing prices cost-calculate as zero, silently zeroing tracked spend - if not _has_valid_token_prices(model.get("pricing")): - continue - model_id = model["id"] - base_model = model.get("base_model") or "" - entry: dict[str, Any] = { - "litellm_provider": FRIENDLI_PROVIDER, - } - - base_key = _find_base_model_entry(base_model, local_data) - if base_key: - base_entry = local_data[base_key] - for k in INHERITABLE_BASE_KEYS: - if k in base_entry: - entry[k] = base_entry[k] - - ctx = model.get("context_length") - if ctx is not None: - entry["max_input_tokens"] = int(ctx) - max_out = model.get("max_completion_tokens") - if max_out is not None: - entry["max_output_tokens"] = int(max_out) - entry["max_tokens"] = int(max_out) - - pricing = _pricing(model.get("pricing", {})) - entry.update(pricing) - entry["supports_prompt_caching"] = "cache_read_input_token_cost" in pricing - - reasoning = model.get("reasoning") is True - entry["supports_reasoning"] = reasoning - if reasoning: - entry["reasoning_effort_levels"] = _reasoning_effort_levels( - model.get("reasoning_options", []) - ) - - func = model.get("functionality", {}) - entry["supports_function_calling"] = func.get("tool_call") is True - entry["supports_parallel_function_calling"] = func.get("parallel_tool_call") is True - is_struct = func.get("structured_output") is True - entry["supports_response_schema"] = is_struct - entry["supports_native_structured_output"] = is_struct - entry["supports_system_messages"] = func.get("system_messages") is True - entry["supports_tool_choice"] = func.get("tool_choice") is True - - entry.update(_modality_flags(model.get("input_modalities", []))) - - entry["mode"] = model.get("mode", "chat") - - desc = model.get("description") - if desc: - entry["comment"] = desc - - dep = model.get("deprecation_date") - if dep: - entry["deprecation_date"] = dep.split("T")[0] - - entry["source"] = FRIENDLI_API_URL - - transformed[f"{FRIENDLI_PROVIDER}/{model_id}"] = entry - return transformed - -# Synchronize local data with remote data -def sync_local_data_with_remote(local_data, remote_data, replace_keys=frozenset()): - # Update existing keys in local_data with values from remote_data - # (replace_keys entries are swapped wholesale so a field the remote catalog - # dropped, e.g. cache pricing, cannot survive as a stale value) - for key in (set(local_data) & set(remote_data)): - if key in replace_keys: - local_data[key] = remote_data[key] - else: - local_data[key].update(remote_data[key]) - - # Add new keys from remote_data to local_data - for key in (set(remote_data) - set(local_data)): - local_data[key] = remote_data[key] - -# Write data to the json file -def write_to_file(file_path, data): - try: - # Open the file in write mode - with open(file_path, "w") as file: - # Dump the data as JSON into the file - json.dump(data, file, indent=4) - print("Values updated successfully.") - except Exception as e: - # Print an error message if writing to file fails - print("Error updating JSON file:", e) - -# Update the existing models and add the missing models for OpenRouter -def transform_openrouter_data(data): - transformed = {} - if not data: - return transformed - for row in data: - # Add the fields 'max_tokens' and 'input_cost_per_token' - obj = { - "max_tokens": row["context_length"], - "input_cost_per_token": float(row["pricing"]["prompt"]), - } - - # Add 'max_output_tokens' as a field if it is not None - if "top_provider" in row and "max_completion_tokens" in row["top_provider"] and row["top_provider"]["max_completion_tokens"] is not None: - obj['max_output_tokens'] = int(row["top_provider"]["max_completion_tokens"]) - - # Add the field 'output_cost_per_token' - obj.update({ - "output_cost_per_token": float(row["pricing"]["completion"]), - }) - - # Add field 'input_cost_per_image' if it exists and is non-zero - if "pricing" in row and "image" in row["pricing"] and float(row["pricing"]["image"]) != 0.0: - obj['input_cost_per_image'] = float(row["pricing"]["image"]) - - # Add the fields 'litellm_provider' and 'mode' - obj.update({ - "litellm_provider": "openrouter", - "mode": "chat" - }) - - # Add the 'supports_vision' field if the modality is 'multimodal' - if row.get('architecture', {}).get('modality') == 'multimodal': - obj['supports_vision'] = True - - # Use a composite key to store the transformed object - transformed[f'openrouter/{row["id"]}'] = obj - - return transformed - -# Update the existing models and add the missing models for Vercel AI Gateway -def transform_vercel_ai_gateway_data(data): - transformed = {} - if not data: - return transformed - for row in data: - # Rows without token pricing or token limits (video/embedding models) previously KeyError'd the whole sync - if any(row.get(k) is None for k in ("context_window", "max_tokens")) or any( - row.get("pricing", {}).get(k) is None for k in ("input", "output") - ): - continue - obj = { - "max_tokens": row["context_window"], - "input_cost_per_token": float(row["pricing"]["input"]), - "output_cost_per_token": float(row["pricing"]["output"]), - 'max_output_tokens': row['max_tokens'], - 'max_input_tokens': row["context_window"], - } - - # Handle cache pricing if available - if "pricing" in row: - if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None: - obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}") - - if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None: - obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}") - - mode = "embedding" if "embedding" in row["id"].lower() else "chat" - - obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode}) - - transformed[f'vercel_ai_gateway/{row["id"]}'] = obj - - return transformed - - -# Load local data from a specified file -def load_local_data(file_path): - try: - # Open the file in read mode - with open(file_path, "r") as file: - # Load and return the JSON data - return json.load(file) - except FileNotFoundError: - # Print an error message if the file is not found - print("File not found:", file_path) - return None - except json.JSONDecodeError as e: - # Print an error message if JSON decoding fails - print("Error decoding JSON:", e) - return None - -def main(): - local_file_path = "model_prices_and_context_window.json" # Path to the local data file - openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data - vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data - - # Load local data from file - local_data = load_local_data(local_file_path) - - # Fetch OpenRouter data - openrouter_data = asyncio.run(fetch_data(openrouter_url)) - # Transform the fetched OpenRouter data - openrouter_data = transform_openrouter_data(openrouter_data) - - # Fetch Vercel AI Gateway data - vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url)) - # Transform the fetched Vercel AI Gateway data - vercel_data = transform_vercel_ai_gateway_data(vercel_data) - - friendli_data = asyncio.run(fetch_data(FRIENDLI_API_URL)) - friendli_data = transform_friendli_data(friendli_data, local_data) - - # Combine both datasets - all_remote_data = {**openrouter_data, **vercel_data, **friendli_data} - - # If both local and openrouter data are available, synchronize and save - if local_data and all_remote_data: - sync_local_data_with_remote(local_data, all_remote_data, replace_keys=frozenset(friendli_data)) - write_to_file(local_file_path, local_data) - else: - print("Failed to fetch model data from either local file or URL.") - -# Entry point of the script -if __name__ == "__main__": - main() diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml deleted file mode 100644 index 838ba35f831..00000000000 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Updates model_prices_and_context_window.json and Create Pull Request - -on: - schedule: - - cron: "0 0 * * 0" # Run every Sundays at midnight - #- cron: "0 0 * * *" # Run daily at midnight - -permissions: - contents: write - pull-requests: write - -jobs: - auto_update_price_and_context_window: - if: github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - name: Set up uv - uses: ./.github/actions/setup-uv-with-retries - with: - version: "0.10.9" - - name: Update JSON Data - run: | - uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py" - - name: Create Pull Request - run: | - git add model_prices_and_context_window.json - git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')" - gh pr create --title "Update model_prices_and_context_window.json file" \ - --body "Automated update for model_prices_and_context_window.json" \ - --head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \ - --base main - env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} diff --git a/.github/workflows/sync-together-ai-models.yml b/.github/workflows/sync-together-ai-models.yml deleted file mode 100644 index 5325510e45e..00000000000 --- a/.github/workflows/sync-together-ai-models.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Sync Together AI model registry - -on: - schedule: - - cron: "30 6 * * *" - workflow_dispatch: - -permissions: - contents: write - pull-requests: write - -jobs: - sync_together_ai_models: - if: github.repository == 'BerriAI/litellm' - runs-on: ubuntu-latest - env: - BASE_BRANCH: ${{ github.event.repository.default_branch }} - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - ref: ${{ env.BASE_BRANCH }} - persist-credentials: false - - name: Set up uv - uses: ./.github/actions/setup-uv-with-retries - with: - version: "0.10.9" - - name: Look for an already-open sync PR - id: existing - run: | - open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \ - --jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')" - echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT" - if [ -n "$open_pr" ]; then - echo "An open sync PR already exists on branch $open_pr; skipping this run." - fi - env: - GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }} - - name: Run the sync - if: steps.existing.outputs.open_pr == '' - run: | - uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md" - env: - TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} - - name: Create a pull request when the registry changed - if: steps.existing.outputs.open_pr == '' - run: | - if git diff --quiet; then - echo "Registry already in sync; no PR needed." - exit 0 - fi - branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git checkout -b "$branch" - git add model_prices_and_context_window.json \ - litellm/model_prices_and_context_window_backup.json - git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')" - gh auth setup-git - git push origin "$branch" - gh pr create --title "feat(models): sync together_ai model registry" \ - --body-file "$RUNNER_TEMP/pr_body.md" \ - --head "$branch" \ - --base "$BASE_BRANCH" - env: - GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }} diff --git a/.github/workflows/test-mcp-dependency-resolution.yml b/.github/workflows/test-mcp-dependency-resolution.yml index 3f0d1260e49..20efb351985 100644 --- a/.github/workflows/test-mcp-dependency-resolution.yml +++ b/.github/workflows/test-mcp-dependency-resolution.yml @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 @@ -132,7 +132,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: diff --git a/.github/workflows/test-merge-smoke.yml b/.github/workflows/test-merge-smoke.yml index 0d1cc6596de..f3f09523491 100644 --- a/.github/workflows/test-merge-smoke.yml +++ b/.github/workflows/test-merge-smoke.yml @@ -33,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] env: LITELLM_LOCAL_MODEL_COST_MAP: "True" steps: @@ -76,6 +76,12 @@ jobs: - name: Import and CLI checks run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py cli + - name: Routing and UUID compatibility + run: >- + $IN_NETNS .venv/bin/python -m pytest + tests/unit/router_strategy/test_complexity_router.py::TestCustomDimensions + tests/unit/test_uuid_helper.py -q + - name: Proxy startup check run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics" diff --git a/.github/workflows/test-rust-db.yml b/.github/workflows/test-rust-db.yml new file mode 100644 index 00000000000..bb88b593d5b --- /dev/null +++ b/.github/workflows/test-rust-db.yml @@ -0,0 +1,106 @@ +name: LiteLLM Rust DB + +on: + push: + paths: + - "litellm-rust/crates/db/**" + - "litellm-rust/Cargo.toml" + - "litellm-rust/Cargo.lock" + - "litellm-rust/clippy.toml" + - "litellm-proxy-extras/litellm_proxy_extras/migrations/**" + - "**/schema.prisma" + - ".cargo/**" + - "rust-toolchain.toml" + - "pyproject.toml" + - "uv.lock" + - ".github/actions/setup-uv-with-retries/**" + - ".github/workflows/test-rust-db.yml" + pull_request: + branches: + - main + - "litellm_**" + paths: + - "litellm-rust/crates/db/**" + - "litellm-rust/Cargo.toml" + - "litellm-rust/Cargo.lock" + - "litellm-rust/clippy.toml" + - "litellm-proxy-extras/litellm_proxy_extras/migrations/**" + - "**/schema.prisma" + - ".cargo/**" + - "rust-toolchain.toml" + - "pyproject.toml" + - "uv.lock" + - ".github/actions/setup-uv-with-retries/**" + - ".github/workflows/test-rust-db.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + db-queries: + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + postgres: + image: postgres:16.6@sha256:557fea37a744d5f4c8faab304b0a90858b53ab119735a88c131fd19dab802f36 + env: + POSTGRES_DB: litellm + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d litellm" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm + SQLX_OFFLINE: "false" + defaults: + run: + working-directory: litellm-rust + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - run: rustup toolchain install --no-self-update + + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: litellm-rust + cache-on-failure: true + + - name: Install Python dependencies + working-directory: . + run: uv sync --frozen --extra extra-proxy + + - name: Deploy Prisma migrations + working-directory: . + run: uv run --no-sync prisma migrate deploy --schema litellm-proxy-extras/litellm_proxy_extras/schema.prisma + + - name: Install SQLx CLI + run: cargo install sqlx-cli --version 0.9.0 --locked --no-default-features -F postgres + + - name: Check SQLx query metadata + working-directory: litellm-rust/crates/db + run: cargo sqlx prepare --check + + - name: Test database queries + run: cargo test -p litellm-db --locked -- --include-ignored diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8e88c0ea15b..70c2d600ba3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -332,12 +332,12 @@ Each translation is isolated in its own file, making it easy to test and modify |--------------|----------|------------------| | `/v1/chat/completions` | Anthropic | `llms/anthropic/chat/transformation.py` | | `/v1/chat/completions` | Bedrock Converse | `llms/bedrock/chat/converse_transformation.py` | -| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` | +| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py` | | `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` | | `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` | | `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` | | `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/pass_through/messages/transformation.py` | -| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` | +| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py` | | `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` | | Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` | @@ -346,7 +346,7 @@ Each translation is isolated in its own file, making it easy to test and modify If `/v1/messages` → Bedrock Converse prompt caching isn't working but Bedrock Invoke works: 1. **Bedrock Converse translation**: `llms/bedrock/chat/converse_transformation.py` -2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` +2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py` 3. Compare how each handles `cache_control` in `transform_request()` ### How translations work diff --git a/Dockerfile b/Dockerfile index 713d7f8f5ab..4cef63c8d71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 diff --git a/backend/Dockerfile b/backend/Dockerfile index 46d65c432bb..1c22f914100 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -124,7 +124,12 @@ ENV HOME=/home/nonroot \ PYTHONUNBUFFERED=1 \ PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries -COPY --from=builder --chown=nonroot:nonroot /app /app +COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv +COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm +COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise +COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend +COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh +COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma COPY --from=builder /opt/prisma /opt/prisma RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ diff --git a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md new file mode 100644 index 00000000000..c79632e9195 --- /dev/null +++ b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md @@ -0,0 +1,76 @@ +# Use your ChatGPT subscription through LiteLLM + +Sign into ChatGPT in Codex or Pi and send model requests through your existing LiteLLM proxy. The client keeps your login and refreshes it. Each request uses that client's ChatGPT account, while a separate LiteLLM virtual key controls gateway access and usage attribution. + +An administrator adds a `chatgpt/` model using the usual model configuration. No ChatGPT login needs to be added to the proxy. This uses the same separate-key convention as provider passthrough, including Claude Code subscription forwarding. + +## Add a model to the proxy + +Reuse your existing proxy. If you need one, follow the [proxy quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start). Merge this entry into its model list and restart the proxy: + +```yaml +model_list: + - model_name: gpt-6-luna + litellm_params: + model: chatgpt/gpt-6-luna + model_info: + mode: responses +``` + +Use a model available to the caller's subscription. The virtual key must allow the exposed model name. Leave saved ChatGPT logins and API credentials unset for a client-only deployment. Explicit cross-provider fallbacks can incur separate API charges; do not configure them for subscription-only traffic. + +## Configure Codex + +Use your existing LiteLLM URL and a virtual key assigned to your gateway user. Export that key as `LITELLM_API_KEY` in the environment that launches Codex. Keep the value out of committed configuration. + +Add this provider to `~/.codex/config.toml`, preserving your other settings: + +```toml +model = "gpt-6-luna" +model_provider = "litellm_subscription" + +[model_providers.litellm_subscription] +name = "LiteLLM subscription" +base_url = "https://litellm.example.com/v1" +wire_api = "responses" +supports_websockets = false +requires_openai_auth = true +env_http_headers = { "x-litellm-api-key" = "LITELLM_API_KEY" } +``` + +Replace `https://litellm.example.com` with your proxy URL. A local proxy normally uses `http://localhost:4000/v1`. Launch Codex, sign into ChatGPT if needed, and send a prompt with a model available to your subscription. The selected model must match the name exposed by the gateway. + +For a managed workspace rollout, use OpenAI's [gateway configuration guide](https://learn.chatgpt.com/docs/enterprise/sign-in-with-chatgpt-through-a-gateway). + +## Native compatibility and Pi + +For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials but does not need a model deployment or resolve model aliases. Restricted keys must allow the upstream model name. + +### Configure Pi + +Sign into the built-in `openai-codex` provider in Pi. Merge this override into `~/.pi/agent/models.json` and export your LiteLLM virtual key as `LITELLM_API_KEY` before launching Pi: + +```json +{ + "providers": { + "openai-codex": { + "baseUrl": "https://litellm.example.com/backend-api/codex", + "headers": { + "x-litellm-api-key": "LITELLM_API_KEY" + } + } + } +} +``` + +Keep Pi's subscription login as the provider credential. The header value names an environment variable, not a key to paste into the file. + +## Verify account usage + +Send a prompt and check the completed request in LiteLLM's Logs page. Its gateway user/key and token counts should match the caller. Give each person their own LiteLLM virtual key for separate attribution. The gateway credential and ChatGPT login are independently authenticated; LiteLLM does not establish that both identities belong to the same person. + +The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key must allow that name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions. + +The proxy forwards the ChatGPT bearer and account header to the fixed OpenAI Codex endpoint. It removes gateway credentials and cookies from the upstream request and redacts credential headers in logging callbacks. The client remains responsible for refresh and reauthentication. Native passthrough returns authentication and quota errors without switching subscriptions. Standard model routes retain the caller login across ChatGPT retries; explicitly configured cross-provider fallbacks use their own credentials. An explicit workspace header is preserved for OpenAI to validate. + +This flow does not connect accounts through the dashboard or pool subscription allowances. Use a Codex build that supports ChatGPT authentication with a custom model provider. The integration forwards HTTP and SSE requests; it does not add native WebSocket or other Codex backend routes. diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 69d9c26b2a8..20f810686a3 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 54bf8c807aa..90bcc523d9e 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,8 +1,8 @@ # syntax=docker/dockerfile:1.7 # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 390a5f5666f..69068cfc8a5 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) ARG PGBOUNCER_VERSION=1.26.0 @@ -140,7 +140,12 @@ ENV HOME=/home/nonroot \ PYTHONUNBUFFERED=1 \ PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries -COPY --from=builder --chown=nonroot:nonroot /app /app +COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv +COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm +COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise +COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway +COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh +COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma COPY --from=builder /opt/prisma /opt/prisma COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index c868d3e4b8a..63137949f38 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -876,6 +876,15 @@ dependencies = [ "time", ] +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -1248,7 +1257,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ "bytes", + "futures-core", "memchr", + "pin-project-lite", + "tokio", + "tokio-util", ] [[package]] @@ -3642,6 +3655,14 @@ dependencies = [ "strum", ] +[[package]] +name = "litellm-circuit-breaker" +version = "0.1.0" +dependencies = [ + "proptest", + "rstest", +] + [[package]] name = "litellm-config" version = "0.1.0" @@ -3655,6 +3676,8 @@ dependencies = [ "strum", "tempfile", "thiserror 2.0.19", + "tracing", + "tracing-subscriber", ] [[package]] @@ -3695,8 +3718,15 @@ dependencies = [ name = "litellm-db" version = "0.1.0" dependencies = [ + "chrono", + "macro_rules_attribute", + "rstest", "serde", + "serde_json", "sqlx", + "strum", + "thiserror 2.0.19", + "tokio", ] [[package]] @@ -3736,6 +3766,7 @@ dependencies = [ "futures-util", "http-body-util", "litellm-auth-types", + "litellm-circuit-breaker", "litellm-config", "litellm-gateway-auth", "litellm-gateway-inference", @@ -3744,6 +3775,8 @@ dependencies = [ "litellm-http", "litellm-inference", "litellm-llms", + "litellm-redis-client", + "litellm-redis-testing", "litellm-secrets", "litellm-tracing", "rstest", @@ -4275,6 +4308,28 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "litellm-redis-client" +version = "0.1.0" +dependencies = [ + "futures-util", + "litellm-circuit-breaker", + "litellm-redis-testing", + "redis", + "rstest", + "thiserror 2.0.19", + "tokio", +] + +[[package]] +name = "litellm-redis-testing" +version = "0.1.0" +dependencies = [ + "testcontainers-modules", + "thiserror 2.0.19", + "tokio", +] + [[package]] name = "litellm-router" version = "0.1.0" @@ -5750,22 +5805,34 @@ dependencies = [ [[package]] name = "redis" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed" +checksum = "7acd809e80afa754c63f586955699d8990efe1c4b6613183ee5f027a11fbc8b5" dependencies = [ + "arc-swap", "arcstr", + "async-lock", + "backon", + "bytes", + "cfg-if", "combine", "crc16", + "futures-channel", + "futures-util", "itoa", + "log", "num-bigint 0.5.1", "percent-encoding", + "pin-project-lite", "rand 0.10.2", "rustls 0.23.42", "rustls-native-certs", "ryu", "sha1_smol", "socket2 0.6.5", + "tokio", + "tokio-rustls 0.26.4", + "tokio-util", "url", "xxhash-rust", ] @@ -8202,18 +8269,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" dependencies = [ "rustls-pki-types", ] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 9e5e0a96b73..dd8d8704370 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -69,6 +69,9 @@ litellm-token-counter-huggingface = { path = "crates/token-counter-huggingface" litellm-token-counter-tiktoken = { path = "crates/token-counter-tiktoken" } litellm-host-python = { path = "crates/host-python" } litellm-python-compat = { path = "crates/python-compat" } +litellm-circuit-breaker = { path = "crates/circuit-breaker" } +litellm-redis-client = { path = "crates/redis-client" } +litellm-redis-testing = { path = "crates/redis-testing" } askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] } tracing = "0.1" @@ -88,6 +91,9 @@ rand = "0.8" macro_rules_attribute = "0.2.3" schemars = "1" reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls", "http2", "stream"] } +redis = "1.8.0" +redis-test = "1.0.4" +testcontainers-modules = "0.15.0" qdrant-client = { version = "1.19.0", default-features = false } uuid = { version = "1", features = ["v4"] } rstest = "0.26.1" @@ -104,6 +110,7 @@ serde_with = { version = "=3.21.0", default-features = false, features = ["std", sha2 = "0.10" syn = { version = "2", default-features = false } sqlx = { version = "0.9.0", default-features = false, features = ["json", "macros", "postgres", "runtime-tokio", "chrono", "tls-rustls-ring-native-roots"] } +chrono = { version = "0.4.45", default-features = false, features = ["std"] } proc-macro2 = "1" quote = "1" subtle = "2" diff --git a/litellm-rust/crates/cache-redis-semantic/Cargo.toml b/litellm-rust/crates/cache-redis-semantic/Cargo.toml index fe5a317cec0..6a5c794bde7 100644 --- a/litellm-rust/crates/cache-redis-semantic/Cargo.toml +++ b/litellm-rust/crates/cache-redis-semantic/Cargo.toml @@ -8,12 +8,12 @@ repository.workspace = true [dependencies] litellm-cache.workspace = true litellm-cache-redis.workspace = true -redis = { version = "1.7.0", features = ["tls-rustls"] } +redis = { workspace = true, features = ["tls-rustls"] } sha2.workspace = true [dev-dependencies] litellm-cache-testing.workspace = true -redis-test = "1.0.4" +redis-test.workspace = true rstest.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/cache-redis/Cargo.toml b/litellm-rust/crates/cache-redis/Cargo.toml index a234286a338..bebfcafd98a 100644 --- a/litellm-rust/crates/cache-redis/Cargo.toml +++ b/litellm-rust/crates/cache-redis/Cargo.toml @@ -7,12 +7,12 @@ repository.workspace = true [dependencies] litellm-cache.workspace = true -redis = { version = "1.7.0", features = ["cluster", "tls-rustls"] } +redis = { workspace = true, features = ["cluster", "tls-rustls"] } r2d2 = "0.8.10" tokio.workspace = true [dev-dependencies] litellm-cache-testing.workspace = true -redis-test = "1.0.4" +redis-test.workspace = true rstest.workspace = true serde_json.workspace = true diff --git a/litellm-rust/crates/cache-response/Cargo.toml b/litellm-rust/crates/cache-response/Cargo.toml index 869c40a12ab..8f4570edc6f 100644 --- a/litellm-rust/crates/cache-response/Cargo.toml +++ b/litellm-rust/crates/cache-response/Cargo.toml @@ -17,8 +17,8 @@ litellm-cache-gcs.workspace = true litellm-http = { workspace = true, features = ["test-support"] } litellm-cache-memory.workspace = true litellm-cache-redis.workspace = true -redis = "1.7.0" -redis-test = "1.0.4" +redis.workspace = true +redis-test.workspace = true rstest.workspace = true tokio.workspace = true wiremock.workspace = true diff --git a/litellm-rust/crates/cache-valkey-semantic/Cargo.toml b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml index da9bb60c853..ec755e61b8c 100644 --- a/litellm-rust/crates/cache-valkey-semantic/Cargo.toml +++ b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml @@ -8,13 +8,13 @@ repository.workspace = true [dependencies] litellm-cache.workspace = true litellm-cache-redis.workspace = true -redis = { version = "1.7.0", features = ["tls-rustls"] } +redis = { workspace = true, features = ["tls-rustls"] } sha2.workspace = true uuid = { version = "1", features = ["v4"] } [dev-dependencies] litellm-cache-testing.workspace = true -redis-test = "1.0.4" +redis-test.workspace = true rstest.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/circuit-breaker/AGENTS.md b/litellm-rust/crates/circuit-breaker/AGENTS.md new file mode 100644 index 00000000000..52ef0ca7435 --- /dev/null +++ b/litellm-rust/crates/circuit-breaker/AGENTS.md @@ -0,0 +1,14 @@ +- `fsm` is a generic state machine runtime and knows nothing about breakers. `breaker` is a generic circuit breaker and knows nothing about Redis. Deciding which errors count as failures belongs to the consumer +- `Machine::step` stays pure. `&self` is config and time comes in on the event, never read inside the machine. The only clock read is `Admission`'s drop, at the call-site boundary +- Transition rules are one `match (state, event)` per machine, with no macro DSL and no `_` arm that would hide a new state from the compiler +- `Shared` counts a transition, bumps the generation and calls the observer only when the state's enum variant changes, matching Python bumping `generation` only when `_state` changes. Data inside a variant (failure counters, `since`) changes without a transition +- The observer runs under the lock so transitions are seen in generation order. Keep it to logging or a metric +- A disabled breaker is `Option::None` at the consumer, not a flag inside the machine. `Config` comes in already resolved, reading `REDIS_CIRCUIT_BREAKER_*` env is the consumer's job + +## Parity with Python `RedisCircuitBreaker` (`litellm/caching/redis_cache.py`) +- The rstest table `matches_the_python_redis_circuit_breaker` was checked case by case against the Python breaker. A behavior change here needs the same check, by driving the Python class with the same steps +- Divergences, each on purpose + - monotonic `Instant` instead of `time.time()`, so a wall clock jump can't open or close the breaker + - a probe `Admission` dropped without an outcome records a hard failure and reopens. Python's probe that dies on `CancelledError` (a `BaseException`) records nothing and leaves the breaker HALF_OPEN for good, and a dropped Rust future is common + - `_failure_count` is dropped, it only fed a log line + - no `enabled` flag, see above diff --git a/litellm-rust/crates/circuit-breaker/Cargo.toml b/litellm-rust/crates/circuit-breaker/Cargo.toml new file mode 100644 index 00000000000..8eca448a2ac --- /dev/null +++ b/litellm-rust/crates/circuit-breaker/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "litellm-circuit-breaker" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] + +[dev-dependencies] +proptest.workspace = true +rstest.workspace = true diff --git a/litellm-rust/crates/circuit-breaker/src/breaker.rs b/litellm-rust/crates/circuit-breaker/src/breaker.rs new file mode 100644 index 00000000000..c9509561b37 --- /dev/null +++ b/litellm-rust/crates/circuit-breaker/src/breaker.rs @@ -0,0 +1,444 @@ +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use crate::{Machine, Observer, Shared}; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Config { + pub failure_threshold: u32, + pub recovery_timeout: Duration, + pub timeout_min_duration: Duration, +} + +impl Default for Config { + fn default() -> Self { + Self { + failure_threshold: 5, + recovery_timeout: Duration::from_secs(60), + timeout_min_duration: Duration::from_secs(5), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Failure { + Hard, + Timeout, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum State { + Closed { + hard: u32, + timeouts: u32, + streak_started: Option, + }, + Open { + since: Instant, + }, + HalfOpen, +} + +impl State { + const CLOSED: Self = Self::Closed { + hard: 0, + timeouts: 0, + streak_started: None, + }; +} + +#[derive(Clone, Copy, Debug)] +pub enum Event { + Admit(Instant), + Succeeded, + Failed(Failure, Instant), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Output { + Admitted, + Probe, + Rejected, + Recorded, +} + +impl Machine for Config { + type State = State; + type Event = Event; + type Output = Output; + + fn step(&self, state: &State, event: Event) -> (State, Output) { + match (state, event) { + (State::Closed { .. }, Event::Admit(_)) => (state.clone(), Output::Admitted), + (State::Open { since }, Event::Admit(now)) + if now.duration_since(*since) > self.recovery_timeout => + { + (State::HalfOpen, Output::Probe) + } + (State::Open { .. } | State::HalfOpen, Event::Admit(_)) => { + (state.clone(), Output::Rejected) + } + (State::Closed { .. } | State::HalfOpen, Event::Succeeded) => { + (State::CLOSED, Output::Recorded) + } + (State::Open { .. }, Event::Succeeded) => (state.clone(), Output::Recorded), + (State::Closed { hard, .. }, Event::Failed(Failure::Hard, now)) => { + let hard = hard + 1; + let next = match hard >= self.failure_threshold { + true => State::Open { since: now }, + false => State::Closed { + hard, + timeouts: 0, + streak_started: None, + }, + }; + (next, Output::Recorded) + } + ( + State::Closed { + hard, + timeouts, + streak_started, + }, + Event::Failed(Failure::Timeout, now), + ) => { + let timeouts = timeouts + 1; + let started = streak_started.unwrap_or(now); + let opens = timeouts >= self.failure_threshold + && now.duration_since(started) >= self.timeout_min_duration; + let next = match opens { + true => State::Open { since: now }, + false => State::Closed { + hard: *hard, + timeouts, + streak_started: Some(started), + }, + }; + (next, Output::Recorded) + } + (State::Open { .. } | State::HalfOpen, Event::Failed(_, now)) => { + (State::Open { since: now }, Output::Recorded) + } + } + } +} + +#[derive(Clone)] +pub struct Breaker(Arc>); + +#[derive(Debug, PartialEq, Eq)] +pub struct Rejected; + +pub struct Admission { + breaker: Breaker, + generation: u64, + probe: bool, + settled: bool, +} + +impl Breaker { + pub fn new(config: Config, observer: Observer) -> Self { + Self(Arc::new(Shared::new(config, State::CLOSED, observer))) + } + + pub fn admit(&self, now: Instant) -> Result { + let (output, generation) = self.0.fire(Event::Admit(now)); + let probe = match output { + Output::Admitted => false, + Output::Probe => true, + Output::Rejected | Output::Recorded => return Err(Rejected), + }; + Ok(Admission { + breaker: self.clone(), + generation, + probe, + settled: false, + }) + } + + pub fn failed(&self, failure: Failure, now: Instant) { + self.0.fire(Event::Failed(failure, now)); + } + + pub fn state(&self) -> State { + self.0.state() + } +} + +impl Admission { + pub fn probe(&self) -> bool { + self.probe + } + + pub fn succeeded(mut self) { + self.settled = true; + self.breaker.0.fire_if(self.generation, Event::Succeeded); + } + + pub fn failed(mut self, failure: Failure, now: Instant) { + self.settled = true; + self.breaker.failed(failure, now); + } +} + +impl Drop for Admission { + fn drop(&mut self) { + if self.probe && !self.settled { + self.breaker.0.fire_if( + self.generation, + Event::Failed(Failure::Hard, Instant::now()), + ); + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::{Duration, Instant}, + }; + + use proptest::prelude::*; + use rstest::rstest; + + use super::{Admission, Breaker, Config, Failure, State}; + use crate::Transition; + + #[derive(Clone, Copy, Debug)] + enum Step { + Fail(f64, Failure), + Admit(f64, bool), + Succeed(usize), + } + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum Phase { + Closed, + Open, + HalfOpen, + } + + use Phase::{Closed as C, HalfOpen as H, Open as O}; + + #[rstest] + fn dropping_an_old_probe_does_not_undo_a_newer_recovery() { + let breaker = Breaker::new( + Config { + failure_threshold: 1, + recovery_timeout: Duration::from_secs(1), + ..Config::default() + }, + Box::new(|_| {}), + ); + let now = Instant::now(); + let pending = breaker.admit(now).unwrap(); + breaker.failed(Failure::Hard, now); + let old_probe = breaker.admit(now + Duration::from_secs(2)).unwrap(); + assert!(old_probe.probe()); + pending.failed(Failure::Hard, now + Duration::from_secs(2)); + let new_probe = breaker.admit(now + Duration::from_secs(4)).unwrap(); + assert!(new_probe.probe()); + new_probe.succeeded(); + assert_eq!(breaker.state(), State::CLOSED); + + drop(old_probe); + + assert_eq!(breaker.state(), State::CLOSED); + assert!(!breaker.admit(now + Duration::from_secs(4)).unwrap().probe()); + } + + fn hard(at: f64) -> Step { + Step::Fail(at, Failure::Hard) + } + + fn timeout(at: f64) -> Step { + Step::Fail(at, Failure::Timeout) + } + + fn five_hard_at_zero() -> Vec { + vec![hard(0.0); 5] + } + + fn phase(state: &State) -> Phase { + match state { + State::Closed { .. } => Phase::Closed, + State::Open { .. } => Phase::Open, + State::HalfOpen => Phase::HalfOpen, + } + } + + fn quiet() -> Breaker { + Breaker::new(Config::default(), Box::new(|_| {})) + } + + #[rstest] + #[case::five_hard_failures_open(vec![hard(0.0), hard(1.0), hard(2.0), hard(3.0), hard(4.0)], vec![C, C, C, C, O])] + #[case::a_success_resets_the_hard_failure_count( + vec![hard(0.0), hard(1.0), hard(2.0), hard(3.0), Step::Admit(4.0, true), Step::Succeed(4), hard(5.0), hard(6.0), hard(7.0), hard(8.0)], + vec![C; 10] + )] + #[case::a_timeout_streak_opens_only_after_the_min_duration( + vec![timeout(0.0), timeout(1.0), timeout(2.0), timeout(3.0), timeout(4.9), timeout(5.0)], + vec![C, C, C, C, C, O] + )] + #[case::a_hard_failure_restarts_the_timeout_streak( + vec![timeout(0.0), timeout(1.0), timeout(2.0), timeout(3.0), hard(4.0), timeout(5.0), timeout(6.0), timeout(7.0), timeout(8.0), timeout(9.9), timeout(10.0)], + vec![C, C, C, C, C, C, C, C, C, C, O] + )] + #[case::the_probe_waits_until_strictly_past_the_recovery_timeout( + [five_hard_at_zero(), vec![Step::Admit(60.0, false), Step::Admit(60.5, true)]].concat(), + vec![C, C, C, C, O, O, H] + )] + #[case::a_probe_success_closes( + [five_hard_at_zero(), vec![Step::Admit(61.0, true), Step::Succeed(5)]].concat(), + vec![C, C, C, C, O, H, C] + )] + #[case::a_probe_failure_reopens_with_a_fresh_timer( + [five_hard_at_zero(), vec![Step::Admit(61.0, true), hard(62.0), Step::Admit(122.0, false), Step::Admit(122.5, true)]].concat(), + vec![C, C, C, C, O, H, O, O, H] + )] + #[case::only_one_probe_at_a_time( + [five_hard_at_zero(), vec![Step::Admit(61.0, true), Step::Admit(61.5, false)]].concat(), + vec![C, C, C, C, O, H, H] + )] + #[case::a_late_failure_while_open_pushes_back_the_probe( + [five_hard_at_zero(), vec![hard(30.0), Step::Admit(61.0, false), Step::Admit(90.5, true)]].concat(), + vec![C, C, C, C, O, O, O, H] + )] + #[case::a_success_from_before_the_breaker_moved_is_ignored( + vec![Step::Admit(0.0, true), hard(1.0), hard(1.0), hard(1.0), hard(1.0), hard(1.0), Step::Admit(62.0, true), Step::Succeed(0), Step::Succeed(6)], + vec![C, C, C, C, C, O, H, H, C] + )] + fn matches_the_python_redis_circuit_breaker( + #[case] steps: Vec, + #[case] expected: Vec, + ) { + let breaker = quiet(); + let t0 = Instant::now(); + let mut admissions: HashMap = HashMap::new(); + + let phases: Vec = steps + .iter() + .enumerate() + .map(|(index, step)| { + match *step { + Step::Fail(at, failure) => { + breaker.failed(failure, t0 + Duration::from_secs_f64(at)) + } + Step::Admit(at, admitted) => { + let admission = breaker.admit(t0 + Duration::from_secs_f64(at)); + assert_eq!(admission.is_ok(), admitted, "step {index}"); + admissions.extend(admission.ok().map(|admission| (index, admission))); + } + Step::Succeed(admitted_at) => { + admissions.remove(&admitted_at).unwrap().succeeded() + } + } + phase(&breaker.state()) + }) + .collect(); + + assert_eq!(phases, expected); + } + + #[rstest] + fn a_dropped_probe_reopens_instead_of_wedging_half_open() { + let breaker = quiet(); + let t0 = Instant::now(); + (0..5).for_each(|_| breaker.failed(Failure::Hard, t0)); + let probe = breaker.admit(t0 + Duration::from_secs(61)).unwrap(); + assert!(probe.probe()); + + drop(probe); + + assert!(matches!(breaker.state(), State::Open { .. })); + } + + #[rstest] + fn a_dropped_regular_call_records_nothing() { + let breaker = quiet(); + let t0 = Instant::now(); + breaker.failed(Failure::Hard, t0); + + drop(breaker.admit(t0).unwrap()); + + assert_eq!( + breaker.state(), + State::Closed { + hard: 1, + timeouts: 0, + streak_started: None + } + ); + } + + #[derive(Clone, Copy, Debug)] + enum Action { + Admit, + SucceedOldest, + FailOldest(Failure), + Fail(Failure), + } + + fn action() -> impl Strategy { + let failure = prop_oneof![Just(Failure::Hard), Just(Failure::Timeout)]; + let action = prop_oneof![ + Just(Action::Admit), + Just(Action::SucceedOldest), + failure.clone().prop_map(Action::FailOldest), + failure.prop_map(Action::Fail), + ]; + (0u64..30_000, action) + } + + proptest! { + #[test] + fn open_never_closes_directly_and_each_half_open_visit_has_one_probe( + actions in prop::collection::vec(action(), 1..200) + ) { + let transitions: Arc>>> = Arc::default(); + let seen = Arc::clone(&transitions); + let config = Config { + failure_threshold: 3, + recovery_timeout: Duration::from_secs(10), + timeout_min_duration: Duration::from_secs(2), + }; + let breaker = Breaker::new(config, Box::new(move |t| seen.lock().unwrap().push(t.clone()))); + let t0 = Instant::now(); + let mut now = t0; + let mut pending: Vec = Vec::new(); + let mut probes_per_visit: Vec = Vec::new(); + + for (advance_ms, action) in actions { + now += Duration::from_millis(advance_ms); + match action { + Action::Admit => { + if let Ok(admission) = breaker.admit(now) { + if admission.probe() { + probes_per_visit.push(1); + } + pending.push(admission); + } + } + Action::SucceedOldest if !pending.is_empty() => pending.remove(0).succeeded(), + Action::FailOldest(failure) if !pending.is_empty() => pending.remove(0).failed(failure, now), + Action::SucceedOldest | Action::FailOldest(_) => {} + Action::Fail(failure) => breaker.failed(failure, now), + } + } + + let transitions = transitions.lock().unwrap(); + let open_to_closed = transitions + .iter() + .filter(|t| matches!((&t.from, &t.to), (State::Open { .. }, State::Closed { .. }))) + .count(); + let half_open_visits = transitions.iter().filter(|t| t.to == State::HalfOpen).count(); + prop_assert_eq!(open_to_closed, 0); + prop_assert_eq!(probes_per_visit.len(), half_open_visits); + } + } +} diff --git a/litellm-rust/crates/circuit-breaker/src/fsm.rs b/litellm-rust/crates/circuit-breaker/src/fsm.rs new file mode 100644 index 00000000000..1beb0f890c8 --- /dev/null +++ b/litellm-rust/crates/circuit-breaker/src/fsm.rs @@ -0,0 +1,174 @@ +use std::{ + fmt::Debug, + mem, + sync::{Mutex, PoisonError}, +}; + +pub trait Machine { + type State: Clone + Debug; + type Event; + type Output; + + fn step(&self, state: &Self::State, event: Self::Event) -> (Self::State, Self::Output); +} + +#[derive(Clone, Debug)] +pub struct Transition { + pub from: S, + pub to: S, + pub generation: u64, +} + +pub type Observer = Box) + Send + Sync>; + +pub struct Shared { + machine: M, + current: Mutex<(M::State, u64)>, + observer: Observer, +} + +impl Shared { + pub fn new(machine: M, initial: M::State, observer: Observer) -> Self { + Self { + machine, + current: Mutex::new((initial, 0)), + observer, + } + } + + pub fn fire(&self, event: M::Event) -> (M::Output, u64) { + let mut current = self.current.lock().unwrap_or_else(PoisonError::into_inner); + self.apply(&mut current, event) + } + + pub fn fire_if(&self, generation: u64, event: M::Event) -> Option { + let mut current = self.current.lock().unwrap_or_else(PoisonError::into_inner); + (current.1 == generation).then(|| self.apply(&mut current, event).0) + } + + pub fn state(&self) -> M::State { + self.current + .lock() + .unwrap_or_else(PoisonError::into_inner) + .0 + .clone() + } + + fn apply(&self, current: &mut (M::State, u64), event: M::Event) -> (M::Output, u64) { + let (to, output) = self.machine.step(¤t.0, event); + if mem::discriminant(&to) != mem::discriminant(¤t.0) { + let from = mem::replace(&mut current.0, to.clone()); + current.1 += 1; + (self.observer)(&Transition { + from, + to, + generation: current.1, + }); + } else { + current.0 = to; + } + (output, current.1) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use rstest::rstest; + + use super::{Machine, Shared, Transition}; + + struct Wrapping(u32); + + #[derive(Clone, Debug, PartialEq, Eq)] + enum Counter { + Low(u32), + High(u32), + } + + enum Tick { + Up, + Stay, + } + + impl Machine for Wrapping { + type State = Counter; + type Event = Tick; + type Output = (); + + fn step(&self, state: &Counter, event: Tick) -> (Counter, ()) { + let value = match (state, event) { + (Counter::Low(_) | Counter::High(_), Tick::Stay) => return (state.clone(), ()), + (Counter::Low(value) | Counter::High(value), Tick::Up) => (value + 1) % self.0, + }; + match value < self.0 / 2 { + true => (Counter::Low(value), ()), + false => (Counter::High(value), ()), + } + } + } + + type Seen = Arc>>>; + + fn recorded(wrap: u32) -> (Shared, Seen) { + let transitions: Seen = Arc::default(); + let seen = Arc::clone(&transitions); + let shared = Shared::new( + Wrapping(wrap), + Counter::Low(0), + Box::new(move |transition| seen.lock().unwrap().push(transition.clone())), + ); + (shared, transitions) + } + + #[rstest] + fn concurrent_events_are_all_applied_and_transitions_are_observed_in_generation_order() { + let (shared, transitions) = recorded(4); + + std::thread::scope(|scope| { + for _ in 0..64 { + scope.spawn(|| { + for _ in 0..100 { + shared.fire(Tick::Up); + } + }); + } + }); + + assert_eq!(shared.state(), Counter::Low(6400 % 4)); + let generations: Vec = transitions + .lock() + .unwrap() + .iter() + .map(|t| t.generation) + .collect(); + assert_eq!(generations, (1..=3200).collect::>()); + } + + #[rstest] + fn a_step_that_keeps_the_variant_neither_counts_nor_notifies() { + let (shared, transitions) = recorded(4); + + let (_, after_inner_change) = shared.fire(Tick::Up); + let (_, after_self_loop) = shared.fire(Tick::Stay); + + assert_eq!(shared.state(), Counter::Low(1)); + assert_eq!((after_inner_change, after_self_loop), (0, 0)); + assert!(transitions.lock().unwrap().is_empty()); + } + + #[rstest] + fn fire_if_skips_an_event_from_an_older_generation() { + let (shared, _) = recorded(4); + shared.fire(Tick::Up); + let (_, generation) = shared.fire(Tick::Up); + + let stale = shared.fire_if(generation - 1, Tick::Up); + let current = shared.fire_if(generation, Tick::Up); + + assert!(stale.is_none()); + assert!(current.is_some()); + assert_eq!(shared.state(), Counter::High(3)); + } +} diff --git a/litellm-rust/crates/circuit-breaker/src/lib.rs b/litellm-rust/crates/circuit-breaker/src/lib.rs new file mode 100644 index 00000000000..e76228122d1 --- /dev/null +++ b/litellm-rust/crates/circuit-breaker/src/lib.rs @@ -0,0 +1,5 @@ +mod breaker; +mod fsm; + +pub use breaker::{Admission, Breaker, Config, Event, Failure, Output, Rejected, State}; +pub use fsm::{Machine, Observer, Shared, Transition}; diff --git a/litellm-rust/crates/config/Cargo.toml b/litellm-rust/crates/config/Cargo.toml index 23d53923362..9746a002bb9 100644 --- a/litellm-rust/crates/config/Cargo.toml +++ b/litellm-rust/crates/config/Cargo.toml @@ -9,11 +9,13 @@ repository.workspace = true litellm-auth-types.workspace = true litellm-router-types.workspace = true serde.workspace = true +serde_json.workspace = true serde_yaml_ng = "0.10.0" strum.workspace = true thiserror.workspace = true +tracing.workspace = true [dev-dependencies] rstest.workspace = true -serde_json.workspace = true tempfile.workspace = true +tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] } diff --git a/litellm-rust/crates/config/src/error.rs b/litellm-rust/crates/config/src/error.rs index 61d19491abc..95c33967cf0 100644 --- a/litellm-rust/crates/config/src/error.rs +++ b/litellm-rust/crates/config/src/error.rs @@ -4,4 +4,11 @@ pub enum Error { Read(#[from] std::io::Error), #[error("invalid YAML config")] Parse(#[from] serde_yaml_ng::Error), + #[error("Redis host does not yet support configured setting {0}")] + RedisSetting(String), + #[error("invalid Redis setting {setting}: {reason}")] + RedisInvalid { + setting: &'static str, + reason: &'static str, + }, } diff --git a/litellm-rust/crates/config/src/lib.rs b/litellm-rust/crates/config/src/lib.rs index 8603ef3279c..b8af6f11d2e 100644 --- a/litellm-rust/crates/config/src/lib.rs +++ b/litellm-rust/crates/config/src/lib.rs @@ -2,6 +2,7 @@ mod error; mod includes; mod mcp; mod model; +mod redis; mod settings; mod value; @@ -13,6 +14,10 @@ pub use error::Error; pub use litellm_router_types::{LitellmParams, Spelled}; pub use mcp::{McpAuth, McpServer, McpTransport}; pub use model::Model; +pub use redis::{ + CacheSettings, Lookup, RedisCluster, RedisEndpoint, RedisHost, RedisNode, RedisSettings, + ResolvedRedis, environment_has_target, +}; pub use settings::{ ClickHouseStoreSettings, GeneralSettings, LiteLlmSettings, RouterSettings, TracingSettings, TracingStoreSettings, diff --git a/litellm-rust/crates/config/src/redis.rs b/litellm-rust/crates/config/src/redis.rs new file mode 100644 index 00000000000..5674f993025 --- /dev/null +++ b/litellm-rust/crates/config/src/redis.rs @@ -0,0 +1,667 @@ +use std::{collections::BTreeMap, fmt, time::Duration}; + +use litellm_auth_types::SecretValue; +use litellm_router_types::Spelled; +use serde::{Deserialize, Deserializer, de::Error as _}; + +use crate::{AdditionalFields, Error, Object, Value}; + +pub type Lookup<'a> = &'a dyn Fn(&str) -> Option; + +const SUPPORTED: &[&str] = &[ + "url", + "host", + "port", + "username", + "password", + "ssl", + "db", + "socket_timeout", + "startup_nodes", + "namespace", +]; + +const IGNORED: &[&str] = &[ + "max_connections", + "single_connection_client", + "health_check_interval", + "socket_keepalive", + "socket_keepalive_options", + "decode_responses", + "encoding", + "encoding_errors", + "charset", + "errors", + "client_name", + "lib_name", + "lib_version", + "sentinel_password", + "gcp_ssl_ca_certs", + "azure_client_id", + "azure_tenant_id", + "azure_client_secret", + "aws_iam_user_name", + "aws_iam_cache_name", + "aws_iam_region", + "aws_iam_serverless", +]; + +const WARN_IGNORED: &[&str] = &[ + "max_connections", + "health_check_interval", + "socket_keepalive", + "socket_keepalive_options", +]; + +const REDIS_PY_KWARGS: &[&str] = &[ + "aws_iam_auth", + "aws_iam_cache_name", + "aws_iam_region", + "aws_iam_serverless", + "aws_iam_user_name", + "azure_client_id", + "azure_client_secret", + "azure_redis_ad_token", + "azure_tenant_id", + "cache", + "cache_config", + "charset", + "client_name", + "db", + "decode_responses", + "encoding", + "encoding_errors", + "errors", + "event_dispatcher", + "gcp_service_account", + "gcp_ssl_ca_certs", + "health_check_interval", + "host", + "lib_name", + "lib_version", + "max_connections", + "password", + "port", + "protocol", + "redis_connect_func", + "retry_on_error", + "retry_on_timeout", + "single_connection_client", + "socket_connect_timeout", + "socket_keepalive", + "socket_keepalive_options", + "socket_timeout", + "ssl", + "ssl_ca_certs", + "ssl_ca_data", + "ssl_ca_path", + "ssl_cert_reqs", + "ssl_certfile", + "ssl_check_hostname", + "ssl_ciphers", + "ssl_keyfile", + "ssl_min_version", + "ssl_ocsp_context", + "ssl_ocsp_expected_cert", + "ssl_password", + "ssl_validate_ocsp", + "ssl_validate_ocsp_stapled", + "unix_socket_path", + "url", + "username", +]; + +const UNSUPPORTED_EXTRA: &[&str] = &[ + "credential_provider", + "sentinel_nodes", + "retry", + "connection_pool", +]; + +const CACHE_ONLY: &[&str] = &[ + "type", + "mode", + "ttl", + "default_in_memory_ttl", + "default_in_redis_ttl", + "supported_call_types", + "similarity_threshold", + "max_messages", + "_backend", +]; + +#[derive(Clone, Default)] +pub struct RedisSettings { + pub url: Option, + pub host: Option>, + pub port: Option>, + pub username: Option, + pub password: Option, + pub ssl: Option>, + pub db: Option>, + pub socket_timeout: Option>, + pub startup_nodes: Option, + pub namespace: Option>, + pub additional_fields: AdditionalFields, + unparsed: DeferredInputs, +} + +impl<'de> Deserialize<'de> for RedisSettings { + fn deserialize>(deserializer: D) -> Result { + let fields = BTreeMap::::deserialize(deserializer)?; + let scalar = |key: &str| fields.get(key).and_then(as_text); + Ok(Self { + url: scalar("url").map(SecretValue::new), + host: scalar("host").map(Spelled::Value), + port: spelled(scalar("port")), + username: scalar("username").map(SecretValue::new), + password: scalar("password").map(SecretValue::new), + ssl: spelled(scalar("ssl")), + db: spelled(scalar("db")), + socket_timeout: spelled(scalar("socket_timeout")), + startup_nodes: fields + .get("startup_nodes") + .filter(|value| !value.is_null()) + .cloned(), + namespace: scalar("namespace").map(Spelled::Value), + additional_fields: cloned(&fields, |key, _| !SUPPORTED.contains(&key)), + unparsed: deferred_inputs(&fields), + }) + } +} + +impl fmt::Debug for RedisSettings { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RedisSettings") + .field("url", &self.url) + .field("host", &self.host) + .field("port", &self.port) + .field("username", &self.username) + .field("password", &self.password) + .field("ssl", &self.ssl) + .field("db", &self.db) + .field("socket_timeout", &self.socket_timeout) + .field( + "startup_nodes", + &self.startup_nodes.as_ref().map(|_| ""), + ) + .field("namespace", &self.namespace) + .field("additional_fields", &self.additional_fields.keys()) + .field("unparsed", &self.unparsed.keys()) + .finish() + } +} + +#[derive(Clone, Debug)] +pub enum CacheSettings { + Redis(Box), + Other(Object), +} + +impl<'de> Deserialize<'de> for CacheSettings { + fn deserialize>(deserializer: D) -> Result { + let fields = BTreeMap::::deserialize(deserializer)?; + let is_redis = matches!(fields.get("type"), Some(Value::String(kind)) if kind == "redis"); + if !is_redis { + return Ok(Self::Other(Object::from_fields(fields))); + } + let mapping = fields + .into_iter() + .map(|(key, value)| (Value::String(key), value)) + .collect(); + serde_yaml_ng::from_value(Value::Mapping(mapping)) + .map(|settings| Self::Redis(Box::new(settings))) + .map_err(D::Error::custom) + } +} + +#[derive(Clone, PartialEq)] +pub struct ResolvedRedis { + pub endpoint: RedisEndpoint, + pub namespace: Option, + pub timeout: Option, +} + +impl fmt::Debug for ResolvedRedis { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ResolvedRedis") + .field("endpoint", &self.endpoint) + .field("namespace", &self.namespace) + .field("timeout", &self.timeout) + .finish() + } +} + +#[derive(Clone, PartialEq)] +pub enum RedisEndpoint { + Url(SecretValue), + Host(RedisHost), + Cluster(RedisCluster), +} + +impl fmt::Debug for RedisEndpoint { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Url(_) => formatter.debug_tuple("Url").field(&"").finish(), + Self::Host(host) => host.fmt(formatter), + Self::Cluster(cluster) => cluster.fmt(formatter), + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct RedisHost { + pub host: String, + pub port: u16, + pub username: Option, + pub password: Option, + pub tls: bool, + pub db: i64, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct RedisCluster { + pub nodes: Vec, + pub username: Option, + pub password: Option, + pub tls: bool, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct RedisNode { + pub host: String, + pub port: u16, +} + +#[derive(Clone)] +struct Entry { + value: Value, + source: String, +} + +type Kwargs = BTreeMap; + +impl RedisSettings { + pub fn resolve(&self, lookup: Lookup<'_>) -> Result { + self.resolve_with(lookup, false) + } + + pub fn resolve_cache(&self, lookup: Lookup<'_>) -> Result { + self.resolve_with(lookup, true) + } + + fn resolve_with(&self, lookup: Lookup<'_>, cache: bool) -> Result { + let block = self.kwargs(cache); + let block: Kwargs = block + .into_iter() + .filter_map(|(key, entry)| { + let value = environment_reference(&entry.value, lookup)?; + Some((key, Entry { value, ..entry })) + }) + .collect(); + let environment = environment_kwargs(lookup); + let names_a_target = ["host", "startup_nodes", "sentinel_nodes"] + .iter() + .any(|key| block.contains_key(*key)); + let drop_environment_url = names_a_target && !block.contains_key("url"); + let merged: Kwargs = environment + .into_iter() + .filter(|(key, _)| !(key == "url" && drop_environment_url)) + .chain(block) + .collect(); + reject_unsupported(&merged, lookup)?; + warn_ignored(&merged); + + let startup_nodes = match merged.get("startup_nodes") { + Some(entry) => Some(cluster_nodes(&entry.value)?), + None => lookup("REDIS_CLUSTER_NODES") + .map(|nodes| cluster_nodes(&Value::String(nodes))) + .transpose()?, + }; + let username = secret_value(&merged, "username"); + let password = secret_value(&merged, "password"); + let tls = bool_value(&merged, "ssl"); + let endpoint = match (startup_nodes, text_value(&merged, "url")) { + (Some(nodes), _) if !nodes.is_empty() => RedisEndpoint::Cluster(RedisCluster { + nodes, + username, + password, + tls, + }), + (_, Some(url)) => RedisEndpoint::Url(SecretValue::new(url)), + _ => RedisEndpoint::Host(RedisHost { + host: text_value(&merged, "host").ok_or(Error::RedisInvalid { + setting: "host", + reason: "either host or url must be specified for redis", + })?, + port: integer(&merged, "port")?.unwrap_or(6379), + username, + password, + tls, + db: integer(&merged, "db")?.unwrap_or(0), + }), + }; + let timeout = merged + .get("socket_timeout") + .and_then(|entry| as_text(&entry.value)) + .map(|seconds| { + seconds + .parse::() + .ok() + .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) + .ok_or(Error::RedisInvalid { + setting: "socket_timeout", + reason: "must be a non-negative number of seconds", + }) + }) + .transpose()?; + Ok(ResolvedRedis { + endpoint, + namespace: text_value(&merged, "namespace"), + timeout, + }) + } + + fn kwargs(&self, cache: bool) -> Kwargs { + let known = [ + ( + "url", + self.url.as_ref().map(|value| value.expose().to_owned()), + ), + ("host", spelled_text(self.host.as_ref())), + ("port", spelled_text(self.port.as_ref())), + ( + "username", + self.username + .as_ref() + .map(|value| value.expose().to_owned()), + ), + ( + "password", + self.password + .as_ref() + .map(|value| value.expose().to_owned()), + ), + ("ssl", spelled_text(self.ssl.as_ref())), + ("db", spelled_text(self.db.as_ref())), + ("socket_timeout", spelled_text(self.socket_timeout.as_ref())), + ("namespace", spelled_text(self.namespace.as_ref())), + ]; + let known = known.into_iter().filter_map(|(key, value)| { + value.map(|value| { + ( + key.to_owned(), + Entry { + value: Value::String(value), + source: key.to_owned(), + }, + ) + }) + }); + let startup_nodes = self.startup_nodes.clone().map(|value| { + ( + "startup_nodes".to_owned(), + Entry { + value, + source: "startup_nodes".to_owned(), + }, + ) + }); + let extras = self.additional_fields.iter().filter_map(|(key, value)| { + if cache && CACHE_ONLY.contains(&key.as_str()) { + return None; + } + let key = match (cache, key.as_str()) { + (true, "redis_startup_nodes") => "startup_nodes".to_owned(), + _ => key.clone(), + }; + Some(( + key.clone(), + Entry { + value: value.clone(), + source: key, + }, + )) + }); + let unparsed = self.unparsed.iter().map(|(key, value)| { + ( + key.clone(), + Entry { + value: value.0.clone(), + source: key.clone(), + }, + ) + }); + extras + .chain(unparsed) + .chain(known) + .chain(startup_nodes) + .collect() + } +} + +pub fn environment_has_target(lookup: Lookup<'_>) -> bool { + [ + "REDIS_HOST", + "REDIS_URL", + "REDIS_CLUSTER_NODES", + "REDIS_SENTINEL_NODES", + ] + .iter() + .any(|name| lookup(name).is_some()) +} + +fn reject_unsupported(merged: &Kwargs, lookup: Lookup<'_>) -> Result<(), Error> { + let active_auth = [ + merged + .get("gcp_service_account") + .map(|entry| entry.source.as_str()) + .or_else(|| lookup("REDIS_GCP_SERVICE_ACCOUNT").map(|_| "REDIS_GCP_SERVICE_ACCOUNT")), + text_value(merged, "azure_redis_ad_token") + .or_else(|| lookup("REDIS_AZURE_AD_TOKEN")) + .filter(|value| value.eq_ignore_ascii_case("true")) + .map(|_| { + merged + .get("azure_redis_ad_token") + .map_or("REDIS_AZURE_AD_TOKEN", |entry| entry.source.as_str()) + }), + bool_value(merged, "aws_iam_auth").then(|| merged["aws_iam_auth"].source.as_str()), + merged + .get("sentinel_nodes") + .map(|entry| entry.source.as_str()) + .or_else(|| lookup("REDIS_SENTINEL_NODES").map(|_| "REDIS_SENTINEL_NODES")), + ]; + if let Some(setting) = active_auth.into_iter().flatten().next() { + return Err(Error::RedisSetting(setting.to_owned())); + } + match merged + .iter() + .find(|(key, _)| is_unsupported(key)) + .map(|(_, entry)| entry.source.clone()) + { + Some(setting) => Err(Error::RedisSetting(setting)), + None => Ok(()), + } +} + +fn warn_ignored(merged: &Kwargs) { + for setting in WARN_IGNORED { + if merged.contains_key(*setting) { + tracing::warn!(setting, "Redis setting is accepted but ignored"); + } + } +} + +fn is_unsupported(key: &str) -> bool { + let known = REDIS_PY_KWARGS.contains(&key) || UNSUPPORTED_EXTRA.contains(&key); + let handled = SUPPORTED.contains(&key) + || IGNORED.contains(&key) + || matches!( + key, + "gcp_service_account" | "azure_redis_ad_token" | "aws_iam_auth" + ); + known && !handled +} + +fn environment_kwargs(lookup: Lookup<'_>) -> Kwargs { + REDIS_PY_KWARGS + .iter() + .filter_map(|key| { + let source = format!("REDIS_{}", key.to_ascii_uppercase()); + let value = lookup(&source)?; + Some(( + (*key).to_owned(), + Entry { + value: Value::String(value), + source, + }, + )) + }) + .collect() +} + +fn environment_reference(value: &Value, lookup: Lookup<'_>) -> Option { + match value { + Value::String(text) => match text.strip_prefix("os.environ/") { + Some(name) => lookup(name).map(Value::String), + None => Some(value.clone()), + }, + Value::Null => None, + _ => Some(value.clone()), + } +} + +fn as_text(value: &Value) -> Option { + match value { + Value::String(text) => Some(text.clone()), + Value::Number(number) => Some(number.to_string()), + Value::Bool(flag) => Some(flag.to_string()), + _ => None, + } +} + +fn text_value(kwargs: &Kwargs, key: &str) -> Option { + kwargs.get(key).and_then(|entry| as_text(&entry.value)) +} + +fn secret_value(kwargs: &Kwargs, key: &str) -> Option { + text_value(kwargs, key).map(SecretValue::new) +} + +fn bool_value(kwargs: &Kwargs, key: &str) -> bool { + match kwargs.get(key).map(|entry| &entry.value) { + Some(Value::Bool(flag)) => *flag, + Some(value) => as_text(value) + .is_some_and(|text| matches!(text.to_ascii_lowercase().as_str(), "true" | "1" | "yes")), + None => false, + } +} + +fn integer(kwargs: &Kwargs, key: &'static str) -> Result, Error> { + kwargs + .get(key) + .and_then(|entry| as_text(&entry.value)) + .map(|text| { + text.parse().map_err(|_| Error::RedisInvalid { + setting: key, + reason: "must be an integer", + }) + }) + .transpose() +} + +const INVALID_NODES: Error = Error::RedisInvalid { + setting: "startup_nodes", + reason: "must be a list of {host, port} objects", +}; + +fn cluster_nodes(value: &Value) -> Result, Error> { + match value { + Value::String(json) => { + let nodes: Vec> = + serde_json::from_str(json).map_err(|_| INVALID_NODES)?; + nodes + .iter() + .map(|node| { + cluster_node( + node.get("host").and_then(json_text), + node.get("port").and_then(json_text), + ) + }) + .collect() + } + Value::Sequence(nodes) => nodes + .iter() + .map(|node| { + cluster_node( + node.get("host").and_then(as_text), + node.get("port").and_then(as_text), + ) + }) + .collect(), + _ => Err(INVALID_NODES), + } +} + +fn cluster_node(host: Option, port: Option) -> Result { + match (host, port.and_then(|port| port.parse().ok())) { + (Some(host), Some(port)) => Ok(RedisNode { host, port }), + _ => Err(INVALID_NODES), + } +} + +fn json_text(value: &serde_json::Value) -> Option { + match value { + serde_json::Value::String(text) => Some(text.clone()), + serde_json::Value::Number(number) => Some(number.to_string()), + _ => None, + } +} + +fn spelled_text(value: Option<&Spelled>) -> Option { + value.map(|value| match value { + Spelled::Value(value) => value.to_string(), + Spelled::Text(value) => value.clone(), + }) +} + +fn spelled(text: Option) -> Option> { + text.map(|text| match text.parse() { + Ok(value) => Spelled::Value(value), + Err(_) => Spelled::Text(text), + }) +} + +fn cloned( + fields: &BTreeMap, + keep: impl Fn(&str, &Value) -> bool, +) -> AdditionalFields { + fields + .iter() + .filter(|(key, value)| keep(key, value)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect() +} + +#[derive(Clone)] +struct DeferredInput(Value); + +type DeferredInputs = BTreeMap; + +fn deferred_inputs(fields: &BTreeMap) -> DeferredInputs { + fields + .iter() + .filter(|(key, value)| is_unparsed(key, value)) + .map(|(key, value)| (key.clone(), DeferredInput(value.clone()))) + .collect() +} + +fn is_unparsed(key: &str, value: &Value) -> bool { + SUPPORTED.contains(&key) + && match key { + "startup_nodes" => value.is_null(), + _ => as_text(value).is_none(), + } +} diff --git a/litellm-rust/crates/config/src/settings.rs b/litellm-rust/crates/config/src/settings.rs index 1094de1638e..9571c4fa5de 100644 --- a/litellm-rust/crates/config/src/settings.rs +++ b/litellm-rust/crates/config/src/settings.rs @@ -3,7 +3,9 @@ use std::fmt; use litellm_auth_types::SecretValue; use serde::Deserialize; -use crate::{AdditionalFields, Object, Spelled, Value, value::one_or_many}; +use crate::{ + AdditionalFields, CacheSettings, Object, RedisSettings, Spelled, Value, value::one_or_many, +}; #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "lowercase")] @@ -76,7 +78,7 @@ pub struct GeneralSettings { pub enable_public_model_hub: bool, pub dangerously_permit_weak_or_unset_master_key: Option, pub plugins: Option>, - pub coordination_redis: Option, + pub coordination_redis: Option, pub mcp_allowed_hosts: Option>, pub mcp_allowed_origins: Box<[String]>, #[serde(flatten)] @@ -220,7 +222,7 @@ pub struct LiteLlmSettings { pub request_timeout: Option>, pub num_retries: Option, pub cache: Option, - pub cache_params: Option, + pub cache_params: Option, #[serde(deserialize_with = "one_or_many")] pub callbacks: Option>, #[serde(deserialize_with = "one_or_many")] diff --git a/litellm-rust/crates/config/src/value.rs b/litellm-rust/crates/config/src/value.rs index 5becdc87b70..667a5afa130 100644 --- a/litellm-rust/crates/config/src/value.rs +++ b/litellm-rust/crates/config/src/value.rs @@ -10,6 +10,10 @@ pub type AdditionalFields = BTreeMap; pub struct Object(BTreeMap); impl Object { + pub(crate) fn from_fields(fields: BTreeMap) -> Self { + Self(fields) + } + pub fn get(&self, key: &str) -> Option<&Value> { self.0.get(key) } diff --git a/litellm-rust/crates/config/tests/redis.rs b/litellm-rust/crates/config/tests/redis.rs new file mode 100644 index 00000000000..ab409541c8e --- /dev/null +++ b/litellm-rust/crates/config/tests/redis.rs @@ -0,0 +1,405 @@ +use std::{ + collections::HashMap, + fmt::Write, + sync::{Arc, Mutex}, +}; + +use litellm_config::{CacheSettings, Config, Error, RedisEndpoint, RedisSettings, ResolvedRedis}; +use rstest::rstest; +use tracing::{Event, Subscriber, field::Visit}; +use tracing_subscriber::{Layer, layer::Context, prelude::*}; + +fn resolve( + yaml: &str, + environment: &[(&str, &str)], +) -> Result { + let environment: HashMap = environment + .iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(); + let config = Config::from_yaml(yaml).unwrap(); + config + .general_settings + .coordination_redis + .as_ref() + .unwrap_or(&RedisSettings::default()) + .resolve(&|name| environment.get(name).cloned()) +} + +fn resolve_cache(yaml: &str) -> Result { + let config = Config::from_yaml(yaml).unwrap(); + match config.litellm_settings.cache_params.unwrap() { + CacheSettings::Redis(settings) => settings.resolve_cache(&|_| None), + CacheSettings::Other(_) => panic!("expected a Redis cache"), + } +} + +fn cluster_ports(resolved: &ResolvedRedis) -> Vec { + match &resolved.endpoint { + RedisEndpoint::Cluster(cluster) => cluster.nodes.iter().map(|node| node.port).collect(), + endpoint => panic!("expected a cluster, got {endpoint:?}"), + } +} + +#[rstest] +#[case::host( + "general_settings:\n coordination_redis:\n host: redis\n port: 6380\n db: 2\n ssl: true\n namespace: app\n socket_timeout: 1.5", + &[], + "host:redis:6380:2:true:app:1500" +)] +#[case::environment_url( + "model_list: []", + &[("REDIS_URL", "redis://environment:6379")], + "url:redis://environment:6379" +)] +#[case::explicit_host_drops_environment_url( + "general_settings:\n coordination_redis:\n host: configured", + &[("REDIS_URL", "redis://environment:6379")], + "host:configured:6379:0:false::none" +)] +#[case::yaml_cluster( + "general_settings:\n coordination_redis:\n startup_nodes:\n - host: node-a\n port: 7000\n - host: node-b\n port: \"7001\"", + &[], + "cluster:node-a:7000,node-b:7001" +)] +#[case::json_environment_cluster( + "model_list: []", + &[("REDIS_CLUSTER_NODES", r#"[{"host":"node-a","port":"7000"},{"host":"node-b","port":7001}]"#)], + "cluster:node-a:7000,node-b:7001" +)] +#[case::missing_reference_keeps_environment_fallback( + "general_settings:\n coordination_redis:\n host: os.environ/MISSING_HOST", + &[("REDIS_HOST", "environment")], + "host:environment:6379:0:false::none" +)] +#[case::configured_values_beat_the_same_environment_values( + "general_settings:\n coordination_redis:\n host: configured\n port: 6380\n db: 3", + &[("REDIS_HOST", "environment"), ("REDIS_PORT", "6390"), ("REDIS_DB", "9")], + "host:configured:6380:3:false::none" +)] +#[case::environment_url_beats_the_environment_host( + "model_list: []", + &[("REDIS_URL", "redis://environment:6379"), ("REDIS_HOST", "ignored")], + "url:redis://environment:6379" +)] +#[case::startup_nodes_beat_a_url( + "general_settings:\n coordination_redis:\n url: redis://single:6379\n startup_nodes:\n - host: node-a\n port: 7000", + &[], + "cluster:node-a:7000" +)] +#[case::environment_references_resolve_the_host_and_port( + "general_settings:\n coordination_redis:\n host: os.environ/COORD_HOST\n port: os.environ/COORD_PORT", + &[("COORD_HOST", "referenced"), ("COORD_PORT", "6400")], + "host:referenced:6400:0:false::none" +)] +#[case::a_null_host_falls_back_to_the_environment( + "general_settings:\n coordination_redis:\n host: null", + &[("REDIS_HOST", "environment")], + "host:environment:6379:0:false::none" +)] +fn resolves_redis_through_the_parsed_config( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] expected: &str, +) { + let resolved = resolve(yaml, environment).unwrap(); + let description = match resolved.endpoint { + RedisEndpoint::Url(url) => format!("url:{}", url.expose()), + RedisEndpoint::Host(host) => format!( + "host:{}:{}:{}:{}:{}:{}", + host.host, + host.port, + host.db, + host.tls, + resolved.namespace.unwrap_or_default(), + resolved + .timeout + .map(|timeout| timeout.as_millis().to_string()) + .unwrap_or_else(|| "none".to_owned()) + ), + RedisEndpoint::Cluster(cluster) => format!( + "cluster:{}", + cluster + .nodes + .iter() + .map(|node| format!("{}:{}", node.host, node.port)) + .collect::>() + .join(",") + ), + }; + + assert_eq!(description, expected); +} + +#[rstest] +#[case::custom_tls_file( + "general_settings:\n coordination_redis:\n host: redis\n ssl_ca_certs: /ca.pem", + &[], + "ssl_ca_certs" +)] +#[case::gcp_auth_from_config( + "general_settings:\n coordination_redis:\n host: redis\n gcp_service_account: account", + &[], + "gcp_service_account" +)] +#[case::gcp_auth_from_environment( + "model_list: []", + &[("REDIS_HOST", "redis"), ("REDIS_GCP_SERVICE_ACCOUNT", "account")], + "REDIS_GCP_SERVICE_ACCOUNT" +)] +#[case::azure_auth_from_config( + "general_settings:\n coordination_redis:\n host: redis\n azure_redis_ad_token: true", + &[], + "azure_redis_ad_token" +)] +#[case::azure_auth_from_environment( + "model_list: []", + &[("REDIS_HOST", "redis"), ("REDIS_AZURE_AD_TOKEN", "true")], + "REDIS_AZURE_AD_TOKEN" +)] +#[case::aws_auth_from_config( + "general_settings:\n coordination_redis:\n host: redis\n aws_iam_auth: true", + &[], + "aws_iam_auth" +)] +#[case::aws_auth_from_environment( + "model_list: []", + &[("REDIS_HOST", "redis"), ("REDIS_AWS_IAM_AUTH", "true")], + "REDIS_AWS_IAM_AUTH" +)] +#[case::sentinel_from_config( + "general_settings:\n coordination_redis:\n sentinel_nodes: [[redis, 26379]]", + &[], + "sentinel_nodes" +)] +#[case::sentinel_from_environment( + "model_list: []", + &[("REDIS_SENTINEL_NODES", "[[\"redis\",26379]]")], + "REDIS_SENTINEL_NODES" +)] +fn rejected_settings_name_their_source( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] expected: &str, +) { + let error = resolve(yaml, environment).unwrap_err(); + + assert!(matches!(error, Error::RedisSetting(setting) if setting == expected)); +} + +#[rstest] +fn an_explicit_host_keeps_environment_credentials() { + let resolved = resolve( + "general_settings:\n coordination_redis:\n host: configured", + &[ + ("REDIS_PASSWORD", "environment-secret"), + ("REDIS_SSL", "True"), + ], + ) + .unwrap(); + + let RedisEndpoint::Host(host) = resolved.endpoint else { + panic!("expected a host endpoint"); + }; + assert_eq!( + host.password.as_ref().map(|password| password.expose()), + Some("environment-secret") + ); + assert!(host.tls); +} + +#[rstest] +#[case::only_a_port("general_settings:\n coordination_redis:\n port: 6379", &[])] +#[case::a_structured_host_with_an_environment_target( + "general_settings:\n coordination_redis:\n host: []", + &[("REDIS_HOST", "environment"), ("REDIS_URL", "redis://environment:6379")] +)] +fn a_block_without_a_usable_target_fails(#[case] yaml: &str, #[case] environment: &[(&str, &str)]) { + let error = resolve(yaml, environment).unwrap_err(); + + assert!(matches!( + error, + Error::RedisInvalid { + setting: "host", + .. + } + )); +} + +#[rstest] +fn the_cache_alias_names_cluster_nodes() { + let resolved = resolve_cache( + "litellm_settings:\n cache_params:\n type: redis\n redis_startup_nodes:\n - host: node-a\n port: 7000", + ) + .unwrap(); + + assert_eq!(cluster_ports(&resolved), vec![7000]); +} + +#[rstest] +fn canonical_startup_nodes_beat_the_cache_alias() { + let resolved = resolve_cache( + "litellm_settings:\n cache_params:\n type: redis\n startup_nodes:\n - host: canonical\n port: 7000\n redis_startup_nodes:\n - host: alias\n port: 7100\n - host: alias\n port: 7101", + ) + .unwrap(); + + assert_eq!(cluster_ports(&resolved), vec![7000]); +} + +#[rstest] +fn a_null_canonical_startup_nodes_setting_shadows_the_cache_alias() { + let resolved = resolve_cache( + "litellm_settings:\n cache_params:\n type: redis\n host: cache\n startup_nodes: null\n redis_startup_nodes:\n - host: alias\n port: 7100", + ) + .unwrap(); + + assert!(matches!(resolved.endpoint, RedisEndpoint::Host(_))); +} + +#[rstest] +fn a_missing_canonical_startup_nodes_reference_shadows_the_cache_alias() { + let resolved = resolve_cache( + "litellm_settings:\n cache_params:\n type: redis\n host: cache\n startup_nodes: os.environ/MISSING_NODES\n redis_startup_nodes:\n - host: alias\n port: 7100", + ) + .unwrap(); + + assert!(matches!(resolved.endpoint, RedisEndpoint::Host(_))); +} + +#[rstest] +fn rejected_environment_settings_never_print_the_value() { + let error = resolve( + "model_list: []", + &[ + ("REDIS_HOST", "redis"), + ("REDIS_SSL_CERT_REQS", "private-option-value"), + ], + ) + .unwrap_err(); + let message = error.to_string(); + + assert!(message.contains("REDIS_SSL_CERT_REQS")); + assert!(!message.contains("private-option-value")); +} + +#[rstest] +#[case::negative("-1")] +#[case::not_a_number("slow")] +fn rejects_bad_socket_timeout(#[case] timeout: &str) { + let yaml = format!( + "general_settings:\n coordination_redis:\n host: redis\n socket_timeout: {timeout}" + ); + let error = resolve(&yaml, &[]).unwrap_err(); + + assert!(matches!( + error, + Error::RedisInvalid { + setting: "socket_timeout", + .. + } + )); +} + +#[rstest] +fn defaults_socket_timeout_for_the_gateway() { + let resolved = resolve( + "general_settings:\n coordination_redis:\n host: redis", + &[], + ) + .unwrap(); + + assert_eq!(resolved.timeout, None); +} + +#[rstest] +fn debug_redacts_redis_secrets_and_url() { + let config = Config::from_yaml( + "general_settings:\n coordination_redis:\n url: redis://user:password@redis:6379\n username: credential-user\n password: credential-password", + ) + .unwrap(); + let settings = config.general_settings.coordination_redis.unwrap(); + let resolved = settings.resolve(&|_| None).unwrap(); + let debug = format!("{settings:?} {resolved:?}"); + + assert!(!debug.contains("redis://user:password")); + assert!(!debug.contains("credential-user")); + assert!(!debug.contains("credential-password")); + assert!(debug.contains("redacted") || debug.contains("REDACTED")); +} + +#[rstest] +fn inactive_redis_cache_keeps_malformed_known_inputs_deferred() { + let config = Config::from_yaml( + "litellm_settings:\n cache: false\n cache_params:\n type: redis\n host: []\n port: invalid\n socket_timeout: invalid", + ) + .unwrap(); + + assert!(matches!( + config.litellm_settings.cache_params, + Some(litellm_config::CacheSettings::Redis(_)) + )); +} + +#[rstest] +fn non_redis_cache_settings_remain_opaque() { + let config = Config::from_yaml( + "litellm_settings:\n cache_params:\n type: local\n custom_option:\n nested: true", + ) + .unwrap(); + + assert!(matches!( + config.litellm_settings.cache_params, + Some(litellm_config::CacheSettings::Other(_)) + )); +} + +#[rstest] +fn legacy_unknown_settings_do_not_change_resolution() { + let resolved = resolve( + "general_settings:\n coordination_redis:\n host: redis\n redis_flush_size: 42\n service_name: primary", + &[], + ) + .unwrap(); + + assert!(matches!(resolved.endpoint, RedisEndpoint::Host(_))); +} + +struct WarningCapture(Arc>>); + +impl Layer for WarningCapture { + fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) { + let mut fields = Fields::default(); + event.record(&mut fields); + self.0.lock().unwrap().push(fields.0); + } +} + +#[derive(Default)] +struct Fields(String); + +impl Visit for Fields { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + write!(self.0, "{}={value:?} ", field.name()).unwrap(); + } + + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + write!(self.0, "{}={value} ", field.name()).unwrap(); + } +} + +#[rstest] +fn ignored_setting_warning_names_the_setting_without_its_value() { + let events = Arc::new(Mutex::new(Vec::new())); + let subscriber = tracing_subscriber::registry().with(WarningCapture(events.clone())); + tracing::subscriber::with_default(subscriber, || { + resolve( + "general_settings:\n coordination_redis:\n host: redis\n max_connections: 47", + &[], + ) + .unwrap(); + }); + let warnings = events.lock().unwrap().join(" "); + + assert!(warnings.contains("max_connections")); + assert!(!warnings.contains("47")); +} diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs index 162d325e4f4..8b63fadf5a6 100644 --- a/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs @@ -51,7 +51,7 @@ mod rules; mod status; mod vertex_ai; -pub use original::{ExceptionFamily, OriginalException}; +pub use original::{ExceptionFamily, OriginalException, TimeoutPhase}; pub use public::{MappedFailure, PublicError, UpstreamResponse}; use rules::{Rule, contains_any, first_match}; @@ -67,7 +67,6 @@ const TIMEOUT_MARKERS: &[&str] = &[ pub struct ExceptionContext { pub model: String, pub custom_llm_provider: String, - pub asynchronous: bool, pub vertex_project: Option, pub vertex_location: Option, pub model_group: Option, @@ -105,11 +104,12 @@ pub fn exception_type( (None, message.clone(), None) } OriginalException::Timeout { + phase, timeout_seconds, elapsed_seconds, } => ( None, - timeout_message(context.asynchronous, *timeout_seconds, *elapsed_seconds), + timeout_message(*phase, *timeout_seconds, *elapsed_seconds), None, ), }; @@ -161,21 +161,16 @@ fn family_rules(family: ExceptionFamily) -> &'static [Rule] { } } -/// The text the Python HTTP handler's timeout carries: the sync and async handlers word it -/// differently. -fn timeout_message( - asynchronous: bool, +/// The text the Python HTTP handler's timeout carries: the phase that expired and the limit +/// httpx enforced for it, both rendered as Python would print them. +pub fn timeout_message( + phase: TimeoutPhase, timeout_seconds: Option, elapsed_seconds: Option, ) -> String { let timeout = python_float(timeout_seconds); - if asynchronous { - let elapsed = - python_float(elapsed_seconds.map(|seconds| (seconds * 1000.0).round() / 1000.0)); - format!("Connection timed out. Timeout passed={timeout}, time taken={elapsed} seconds") - } else { - format!("Connection timed out after {timeout} seconds.") - } + let elapsed = python_float(elapsed_seconds.map(|seconds| (seconds * 1000.0).round() / 1000.0)); + format!("{phase} timed out. Timeout passed={timeout}, time taken={elapsed} seconds") } fn python_float(value: Option) -> String { @@ -356,6 +351,7 @@ mod tests { #[test] fn a_handler_timeout_is_a_408_without_a_response() { let original = OriginalException::Timeout { + phase: TimeoutPhase::Read, timeout_seconds: Some(0.5), elapsed_seconds: Some(0.5031), }; @@ -363,7 +359,7 @@ mod tests { mapped("mistral", &original), MappedFailure { error: PublicError::Timeout { status: 408 }, - message: "MistralException - Connection timed out after 0.5 seconds.".into(), + message: "MistralException - Read timed out. Timeout passed=0.5, time taken=0.503 seconds".into(), upstream: None, debug_info: DEBUG.into(), } @@ -440,39 +436,44 @@ mod tests { } #[rstest::rstest] - #[case::sync( - false, + #[case::connect(TimeoutPhase::Connect, "Connection timed out.")] + #[case::read(TimeoutPhase::Read, "Read timed out.")] + #[case::write(TimeoutPhase::Write, "Write timed out.")] + #[case::pool(TimeoutPhase::Pool, "Connection pool timed out.")] + fn timeout_text_names_the_phase_that_expired( + #[case] phase: TimeoutPhase, + #[case] expected: &str, + ) { + assert_eq!( + timeout_message(phase, Some(0.5), Some(0.5031)), + format!("{expected} Timeout passed=0.5, time taken=0.503 seconds") + ); + } + + #[rstest::rstest] + #[case::rounds_the_elapsed_time( Some(0.5), Some(0.5031), - "Connection timed out after 0.5 seconds." - )] - #[case::async_rounds_the_elapsed_time( - true, - Some(0.5), - Some(0.5031), - "Connection timed out. Timeout passed=0.5, time taken=0.503 seconds" + "Timeout passed=0.5, time taken=0.503 seconds" )] #[case::whole_seconds_keep_a_decimal( - true, Some(600.0), Some(2.0), - "Connection timed out. Timeout passed=600.0, time taken=2.0 seconds" + "Timeout passed=600.0, time taken=2.0 seconds" )] #[case::unknown_values_render_as_none( - true, None, None, - "Connection timed out. Timeout passed=None, time taken=None seconds" + "Timeout passed=None, time taken=None seconds" )] - fn timeout_text_follows_the_delivery_mode( - #[case] asynchronous: bool, + fn timeout_text_renders_durations_as_python_floats( #[case] timeout_seconds: Option, #[case] elapsed_seconds: Option, #[case] expected: &str, ) { assert_eq!( - timeout_message(asynchronous, timeout_seconds, elapsed_seconds), - expected + timeout_message(TimeoutPhase::Read, timeout_seconds, elapsed_seconds), + format!("Read timed out. {expected}") ); } diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs index 4de67e6e7b5..6517b093bbd 100644 --- a/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs @@ -10,6 +10,7 @@ pub enum OriginalException { message: String, }, Timeout { + phase: TimeoutPhase, timeout_seconds: Option, elapsed_seconds: Option, }, @@ -20,6 +21,19 @@ pub enum OriginalException { }, } +/// The httpx timeout that expired, spelled the way the Python HTTP handler names it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::VariantArray)] +pub enum TimeoutPhase { + #[strum(serialize = "Connection")] + Connect, + #[strum(serialize = "Read")] + Read, + #[strum(serialize = "Write")] + Write, + #[strum(serialize = "Connection pool")] + Pool, +} + /// Which provider-specific text rules apply before the shared status table. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ExceptionFamily { diff --git a/litellm-rust/crates/db-testing/Cargo.toml b/litellm-rust/crates/db-testing/Cargo.toml index 3ba71fbf644..457c0f2ad87 100644 --- a/litellm-rust/crates/db-testing/Cargo.toml +++ b/litellm-rust/crates/db-testing/Cargo.toml @@ -7,7 +7,7 @@ repository.workspace = true [dependencies] sqlx = { workspace = true, features = ["migrate"] } -testcontainers-modules = { version = "0.15.0", features = ["postgres"] } +testcontainers-modules = { workspace = true, features = ["postgres"] } thiserror.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/db/.sqlx/query-085edfb9af2a914466c75b447bc7bb2d97532a9cd461cc5d8da1e1090560ca1f.json b/litellm-rust/crates/db/.sqlx/query-085edfb9af2a914466c75b447bc7bb2d97532a9cd461cc5d8da1e1090560ca1f.json new file mode 100644 index 00000000000..a415666d0be --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-085edfb9af2a914466c75b447bc7bb2d97532a9cd461cc5d8da1e1090560ca1f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext($1)) IS NULL AS \"locked!\"\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "locked!", + "type_info": "Bool", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "085edfb9af2a914466c75b447bc7bb2d97532a9cd461cc5d8da1e1090560ca1f" +} diff --git a/litellm-rust/crates/db/.sqlx/query-32f3c07ced96324b535bbc703c1acf6d06eb22facdf478f88107c9dccdac326b.json b/litellm-rust/crates/db/.sqlx/query-32f3c07ced96324b535bbc703c1acf6d06eb22facdf478f88107c9dccdac326b.json new file mode 100644 index 00000000000..c4eb1737d85 --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-32f3c07ced96324b535bbc703c1acf6d06eb22facdf478f88107c9dccdac326b.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO \"LiteLLM_SpendLogs\" (\n request_id, call_type, api_key, spend, total_tokens, prompt_tokens,\n completion_tokens, \"startTime\", \"endTime\", \"completionStartTime\",\n request_duration_ms, model, model_id, model_group, custom_llm_provider, api_base,\n \"user\", metadata, cache_hit, cache_key, request_tags, team_id, organization_id,\n end_user, requester_ip_address, messages, response, proxy_server_request,\n session_id, status, litellm_call_id\n)\nSELECT l.request_id, l.call_type, l.api_key, l.spend, l.total_tokens, l.prompt_tokens,\n l.completion_tokens, l.start_time::timestamp(3), l.end_time::timestamp(3),\n l.completion_start_time::timestamp(3), l.request_duration_ms, l.model,\n l.model_id, l.model_group, l.custom_llm_provider, l.api_base, l.user_id,\n l.metadata, l.cache_hit, l.cache_key, '[]'::jsonb, l.team_id,\n l.organization_id, l.end_user, l.requester_ip_address, '{}'::jsonb,\n '{}'::jsonb, '{}'::jsonb, l.session_id, l.status, l.litellm_call_id\nFROM UNNEST($1::text[], $2::text[], $3::text[], $4::float8[], $5::int4[], $6::int4[],\n $7::int4[], $8::timestamp[], $9::timestamp[], $10::timestamp[], $11::int4[], $12::text[],\n $13::text[], $14::text[], $15::text[], $16::text[], $17::text[],\n $18::jsonb[], $19::text[], $20::text[], $21::text[], $22::text[],\n $23::text[], $24::text[], $25::text[], $26::text[], $27::text[])\n AS l(request_id, call_type, api_key, spend, total_tokens, prompt_tokens,\n completion_tokens, start_time, end_time, completion_start_time,\n request_duration_ms, model, model_id, model_group, custom_llm_provider,\n api_base, user_id, metadata, cache_hit, cache_key, team_id, organization_id,\n end_user, requester_ip_address, session_id, status, litellm_call_id)\nON CONFLICT DO NOTHING\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "Float8Array", + "Int4Array", + "Int4Array", + "Int4Array", + "TimestampArray", + "TimestampArray", + "TimestampArray", + "Int4Array", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "JsonbArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "32f3c07ced96324b535bbc703c1acf6d06eb22facdf478f88107c9dccdac326b" +} diff --git a/litellm-rust/crates/db/.sqlx/query-345c02595f55a7f0274ef7214b02288e0f2c1f12a2a79ed53f0002826724cd0a.json b/litellm-rust/crates/db/.sqlx/query-345c02595f55a7f0274ef7214b02288e0f2c1f12a2a79ed53f0002826724cd0a.json new file mode 100644 index 00000000000..123ff15ade2 --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-345c02595f55a7f0274ef7214b02288e0f2c1f12a2a79ed53f0002826724cd0a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO \"LiteLLM_EndUserTable\" (user_id, spend, blocked)\nSELECT id, SUM(cost), false FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\nGROUP BY id\nORDER BY id\nON CONFLICT (user_id) DO UPDATE\nSET spend = \"LiteLLM_EndUserTable\".spend + EXCLUDED.spend\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Float8Array" + ] + }, + "nullable": [] + }, + "hash": "345c02595f55a7f0274ef7214b02288e0f2c1f12a2a79ed53f0002826724cd0a" +} diff --git a/litellm-rust/crates/db/.sqlx/query-377d063cd0fc3a265d07d01f5aa64311d25428fd393bac4250891f148597e92e.json b/litellm-rust/crates/db/.sqlx/query-377d063cd0fc3a265d07d01f5aa64311d25428fd393bac4250891f148597e92e.json new file mode 100644 index 00000000000..734935466ff --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-377d063cd0fc3a265d07d01f5aa64311d25428fd393bac4250891f148597e92e.json @@ -0,0 +1,78 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT e.spend AS \"spend?\",\n e.blocked AS \"blocked?\",\n b.max_budget AS \"max_budget?\",\n b.rpm_limit AS \"rpm_limit?\",\n b.tpm_limit AS \"tpm_limit?\"\nFROM (SELECT 1) AS one\nLEFT JOIN \"LiteLLM_EndUserTable\" AS e ON e.user_id = $1\nLEFT JOIN \"LiteLLM_BudgetTable\" AS assigned ON assigned.budget_id = e.budget_id\nLEFT JOIN \"LiteLLM_BudgetTable\" AS key_budget ON key_budget.budget_id = $3\nLEFT JOIN \"LiteLLM_BudgetTable\" AS b ON b.budget_id = COALESCE(assigned.budget_id, key_budget.budget_id, $2)\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "spend?", + "type_info": "Float8", + "origin": { + "Table": { + "table": "\"LiteLLM_EndUserTable\"", + "name": "spend" + } + } + }, + { + "ordinal": 1, + "name": "blocked?", + "type_info": "Bool", + "origin": { + "Table": { + "table": "\"LiteLLM_EndUserTable\"", + "name": "blocked" + } + } + }, + { + "ordinal": 2, + "name": "max_budget?", + "type_info": "Float8", + "origin": { + "Table": { + "table": "\"LiteLLM_BudgetTable\"", + "name": "max_budget" + } + } + }, + { + "ordinal": 3, + "name": "rpm_limit?", + "type_info": "Int8", + "origin": { + "Table": { + "table": "\"LiteLLM_BudgetTable\"", + "name": "rpm_limit" + } + } + }, + { + "ordinal": 4, + "name": "tpm_limit?", + "type_info": "Int8", + "origin": { + "Table": { + "table": "\"LiteLLM_BudgetTable\"", + "name": "tpm_limit" + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + true, + true, + true, + true, + true + ] + }, + "hash": "377d063cd0fc3a265d07d01f5aa64311d25428fd393bac4250891f148597e92e" +} diff --git a/litellm-rust/crates/db/.sqlx/query-3829a383d0ae50578f0fd75da3910731a0cacc1ae5b92eec32967f539948f563.json b/litellm-rust/crates/db/.sqlx/query-3829a383d0ae50578f0fd75da3910731a0cacc1ae5b92eec32967f539948f563.json new file mode 100644 index 00000000000..e85c0986a60 --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-3829a383d0ae50578f0fd75da3910731a0cacc1ae5b92eec32967f539948f563.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH added AS (\n SELECT id, SUM(cost) AS cost\n FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\n GROUP BY id\n), locked AS (\n SELECT t.token FROM \"LiteLLM_VerificationToken\" t\n WHERE t.token IN (SELECT id FROM added)\n ORDER BY t.token FOR UPDATE\n)\nUPDATE \"LiteLLM_VerificationToken\" t\nSET spend = t.spend + added.cost,\n total_spend = t.total_spend + added.cost,\n last_active = (clock_timestamp() AT TIME ZONE 'UTC'),\n updated_at = (clock_timestamp() AT TIME ZONE 'UTC')\nFROM added\nWHERE t.token = added.id AND t.token IN (SELECT token FROM locked)\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Float8Array" + ] + }, + "nullable": [] + }, + "hash": "3829a383d0ae50578f0fd75da3910731a0cacc1ae5b92eec32967f539948f563" +} diff --git a/litellm-rust/crates/db/.sqlx/query-43f2e12a3ffb1de29b5b031a1173b87a20a2664b0bef3616631d2c24487c1e54.json b/litellm-rust/crates/db/.sqlx/query-43f2e12a3ffb1de29b5b031a1173b87a20a2664b0bef3616631d2c24487c1e54.json new file mode 100644 index 00000000000..a758ae5126a --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-43f2e12a3ffb1de29b5b031a1173b87a20a2664b0bef3616631d2c24487c1e54.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH added AS (\n SELECT id, SUM(cost) AS cost\n FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\n GROUP BY id\n), locked AS (\n SELECT u.user_id FROM \"LiteLLM_UserTable\" u\n WHERE u.user_id IN (SELECT id FROM added)\n ORDER BY u.user_id FOR UPDATE\n)\nUPDATE \"LiteLLM_UserTable\" u\nSET spend = u.spend + added.cost,\n updated_at = (NOW() AT TIME ZONE 'UTC')\nFROM added\nWHERE u.user_id = added.id AND u.user_id IN (SELECT user_id FROM locked)\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Float8Array" + ] + }, + "nullable": [] + }, + "hash": "43f2e12a3ffb1de29b5b031a1173b87a20a2664b0bef3616631d2c24487c1e54" +} diff --git a/litellm-rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json b/litellm-rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json new file mode 100644 index 00000000000..7eef5278cb5 --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT set_config('lock_timeout', '5000ms', true)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "set_config", + "type_info": "Text", + "origin": "Expression" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4" +} diff --git a/litellm-rust/crates/db/.sqlx/query-9c51401a8907f818e96da28050b74dca981ff954452b35ffbace3e022035b416.json b/litellm-rust/crates/db/.sqlx/query-9c51401a8907f818e96da28050b74dca981ff954452b35ffbace3e022035b416.json new file mode 100644 index 00000000000..9886a9505d8 --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-9c51401a8907f818e96da28050b74dca981ff954452b35ffbace3e022035b416.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO \"LiteLLM_DailyTeamSpend\" (\n id, team_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,\n completion_tokens, api_requests, successful_requests, failed_requests,\n cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,\n timed_requests, spend, updated_at\n)\nSELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,\n r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,\n r.completion_tokens, r.api_requests, r.successful_requests,\n r.failed_requests, r.cache_read, r.cache_creation, r.response_time,\n r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')\nFROM UNNEST(\n $1::text[], $2::text[], $3::text[], $4::text[], $5::text[],\n $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],\n $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],\n $16::int8[], $17::float8[], $18::text[]\n) AS r(\n entity_id, date, api_key, model, custom_llm_provider, endpoint,\n model_group, prompt_tokens, completion_tokens, api_requests,\n successful_requests, failed_requests, cache_read, cache_creation,\n response_time, timed_requests, spend, mcp_namespaced_tool_name\n)\nORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,\n r.mcp_namespaced_tool_name, r.endpoint\nON CONFLICT (\n team_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint\n) DO UPDATE SET\n prompt_tokens = \"LiteLLM_DailyTeamSpend\".prompt_tokens + EXCLUDED.prompt_tokens,\n completion_tokens = \"LiteLLM_DailyTeamSpend\".completion_tokens + EXCLUDED.completion_tokens,\n api_requests = \"LiteLLM_DailyTeamSpend\".api_requests + EXCLUDED.api_requests,\n successful_requests = \"LiteLLM_DailyTeamSpend\".successful_requests + EXCLUDED.successful_requests,\n failed_requests = \"LiteLLM_DailyTeamSpend\".failed_requests + EXCLUDED.failed_requests,\n cache_read_input_tokens = \"LiteLLM_DailyTeamSpend\".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,\n cache_creation_input_tokens = \"LiteLLM_DailyTeamSpend\".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,\n total_response_time_ms = \"LiteLLM_DailyTeamSpend\".total_response_time_ms + EXCLUDED.total_response_time_ms,\n timed_requests = \"LiteLLM_DailyTeamSpend\".timed_requests + EXCLUDED.timed_requests,\n spend = \"LiteLLM_DailyTeamSpend\".spend + EXCLUDED.spend,\n updated_at = (NOW() AT TIME ZONE 'UTC')\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Float8Array", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "9c51401a8907f818e96da28050b74dca981ff954452b35ffbace3e022035b416" +} diff --git a/litellm-rust/crates/db/.sqlx/query-b0eb48b795bdaf728c79e986d2027df60ff71d8fed05b53be4458b335af88147.json b/litellm-rust/crates/db/.sqlx/query-b0eb48b795bdaf728c79e986d2027df60ff71d8fed05b53be4458b335af88147.json new file mode 100644 index 00000000000..b333ec814a1 --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-b0eb48b795bdaf728c79e986d2027df60ff71d8fed05b53be4458b335af88147.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO \"LiteLLM_TeamMembership\" (user_id, team_id, spend, total_spend)\nSELECT member.user_id, member.team_id, member.cost, member.cost\nFROM jsonb_to_recordset($1::jsonb) AS member(user_id text, team_id text, cost float8)\nWHERE EXISTS (\n SELECT 1 FROM \"LiteLLM_TeamTable\" t\n WHERE t.team_id = member.team_id\n AND t.members_with_roles @> jsonb_build_array(jsonb_build_object('user_id', member.user_id))\n)\n OR EXISTS (\n SELECT 1 FROM \"LiteLLM_TeamMembership\" m\n WHERE m.user_id = member.user_id AND m.team_id = member.team_id\n)\nON CONFLICT (user_id, team_id) DO UPDATE\nSET spend = \"LiteLLM_TeamMembership\".spend + EXCLUDED.spend,\n total_spend = \"LiteLLM_TeamMembership\".total_spend + EXCLUDED.total_spend\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "b0eb48b795bdaf728c79e986d2027df60ff71d8fed05b53be4458b335af88147" +} diff --git a/litellm-rust/crates/db/.sqlx/query-bec3b53762af021a118d63b841b8c2ebd7fc9fcbf19c1c25aa131a1053d96cfe.json b/litellm-rust/crates/db/.sqlx/query-bec3b53762af021a118d63b841b8c2ebd7fc9fcbf19c1c25aa131a1053d96cfe.json new file mode 100644 index 00000000000..6c56868e48e --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-bec3b53762af021a118d63b841b8c2ebd7fc9fcbf19c1c25aa131a1053d96cfe.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO \"LiteLLM_DailyUserSpend\" (\n id, user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,\n completion_tokens, api_requests, successful_requests, failed_requests,\n cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,\n timed_requests, spend, updated_at\n)\nSELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,\n r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,\n r.completion_tokens, r.api_requests, r.successful_requests,\n r.failed_requests, r.cache_read, r.cache_creation, r.response_time,\n r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')\nFROM UNNEST(\n $1::text[], $2::text[], $3::text[], $4::text[], $5::text[],\n $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],\n $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],\n $16::int8[], $17::float8[], $18::text[]\n) AS r(\n entity_id, date, api_key, model, custom_llm_provider, endpoint,\n model_group, prompt_tokens, completion_tokens, api_requests,\n successful_requests, failed_requests, cache_read, cache_creation,\n response_time, timed_requests, spend, mcp_namespaced_tool_name\n)\nORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,\n r.mcp_namespaced_tool_name, r.endpoint\nON CONFLICT (\n user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint\n) DO UPDATE SET\n prompt_tokens = \"LiteLLM_DailyUserSpend\".prompt_tokens + EXCLUDED.prompt_tokens,\n completion_tokens = \"LiteLLM_DailyUserSpend\".completion_tokens + EXCLUDED.completion_tokens,\n api_requests = \"LiteLLM_DailyUserSpend\".api_requests + EXCLUDED.api_requests,\n successful_requests = \"LiteLLM_DailyUserSpend\".successful_requests + EXCLUDED.successful_requests,\n failed_requests = \"LiteLLM_DailyUserSpend\".failed_requests + EXCLUDED.failed_requests,\n cache_read_input_tokens = \"LiteLLM_DailyUserSpend\".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,\n cache_creation_input_tokens = \"LiteLLM_DailyUserSpend\".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,\n total_response_time_ms = \"LiteLLM_DailyUserSpend\".total_response_time_ms + EXCLUDED.total_response_time_ms,\n timed_requests = \"LiteLLM_DailyUserSpend\".timed_requests + EXCLUDED.timed_requests,\n spend = \"LiteLLM_DailyUserSpend\".spend + EXCLUDED.spend,\n updated_at = (NOW() AT TIME ZONE 'UTC')\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Float8Array", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "bec3b53762af021a118d63b841b8c2ebd7fc9fcbf19c1c25aa131a1053d96cfe" +} diff --git a/litellm-rust/crates/db/.sqlx/query-c3e5542906e89995fae2a16eca1edd37a575d641bb68d91e61f5123173c275ce.json b/litellm-rust/crates/db/.sqlx/query-c3e5542906e89995fae2a16eca1edd37a575d641bb68d91e61f5123173c275ce.json new file mode 100644 index 00000000000..250971b053a --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-c3e5542906e89995fae2a16eca1edd37a575d641bb68d91e61f5123173c275ce.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH added AS (\n SELECT id, SUM(cost) AS cost\n FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\n GROUP BY id\n),\nlocked AS (\n SELECT t.team_id FROM \"LiteLLM_TeamTable\" t\n WHERE t.team_id IN (SELECT id FROM added)\n ORDER BY t.team_id FOR UPDATE\n)\nUPDATE \"LiteLLM_TeamTable\" t\nSET spend = t.spend + added.cost,\n updated_at = (NOW() AT TIME ZONE 'UTC')\nFROM added\nWHERE t.team_id = added.id AND t.team_id IN (SELECT team_id FROM locked)\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Float8Array" + ] + }, + "nullable": [] + }, + "hash": "c3e5542906e89995fae2a16eca1edd37a575d641bb68d91e61f5123173c275ce" +} diff --git a/litellm-rust/crates/db/.sqlx/query-f207ff0f5310dea0833361d55993bcab69ff6ad3555a3926537586f5d81d933d.json b/litellm-rust/crates/db/.sqlx/query-f207ff0f5310dea0833361d55993bcab69ff6ad3555a3926537586f5d81d933d.json new file mode 100644 index 00000000000..b2635c549ac --- /dev/null +++ b/litellm-rust/crates/db/.sqlx/query-f207ff0f5310dea0833361d55993bcab69ff6ad3555a3926537586f5d81d933d.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO \"LiteLLM_DailyEndUserSpend\" (\n id, end_user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,\n completion_tokens, api_requests, successful_requests, failed_requests,\n cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,\n timed_requests, spend, updated_at\n)\nSELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,\n r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,\n r.completion_tokens, r.api_requests, r.successful_requests,\n r.failed_requests, r.cache_read, r.cache_creation, r.response_time,\n r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')\nFROM UNNEST(\n $1::text[], $2::text[], $3::text[], $4::text[], $5::text[],\n $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],\n $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],\n $16::int8[], $17::float8[], $18::text[]\n) AS r(\n entity_id, date, api_key, model, custom_llm_provider, endpoint,\n model_group, prompt_tokens, completion_tokens, api_requests,\n successful_requests, failed_requests, cache_read, cache_creation,\n response_time, timed_requests, spend, mcp_namespaced_tool_name\n)\nORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,\n r.mcp_namespaced_tool_name, r.endpoint\nON CONFLICT (\n end_user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint\n) DO UPDATE SET\n prompt_tokens = \"LiteLLM_DailyEndUserSpend\".prompt_tokens + EXCLUDED.prompt_tokens,\n completion_tokens = \"LiteLLM_DailyEndUserSpend\".completion_tokens + EXCLUDED.completion_tokens,\n api_requests = \"LiteLLM_DailyEndUserSpend\".api_requests + EXCLUDED.api_requests,\n successful_requests = \"LiteLLM_DailyEndUserSpend\".successful_requests + EXCLUDED.successful_requests,\n failed_requests = \"LiteLLM_DailyEndUserSpend\".failed_requests + EXCLUDED.failed_requests,\n cache_read_input_tokens = \"LiteLLM_DailyEndUserSpend\".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,\n cache_creation_input_tokens = \"LiteLLM_DailyEndUserSpend\".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,\n total_response_time_ms = \"LiteLLM_DailyEndUserSpend\".total_response_time_ms + EXCLUDED.total_response_time_ms,\n timed_requests = \"LiteLLM_DailyEndUserSpend\".timed_requests + EXCLUDED.timed_requests,\n spend = \"LiteLLM_DailyEndUserSpend\".spend + EXCLUDED.spend,\n updated_at = (NOW() AT TIME ZONE 'UTC')\n", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Float8Array", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "f207ff0f5310dea0833361d55993bcab69ff6ad3555a3926537586f5d81d933d" +} diff --git a/litellm-rust/crates/db/AGENTS.md b/litellm-rust/crates/db/AGENTS.md new file mode 100644 index 00000000000..1e08317230e --- /dev/null +++ b/litellm-rust/crates/db/AGENTS.md @@ -0,0 +1,25 @@ +# Database crate + +Keep each static query in `queries/` and call it through a typed SQLx query function. Rust does not create or migrate the database + +- Own reusable row projections in `src/` and export those consumed by other crates or integration tests +- Apply `#[macro_rules_attribute::apply(crate::db_row)]` to row types for shared derives, without adding serialization or schema derives automatically +- Apply `#[macro_rules_attribute::apply(crate::db_input)]` to write inputs, adding `Default` only where supported +- Keep function tests inline in their owning query module and reuse the strict database fixture in `test_support` +- Transaction-local setup in `begin` is the approved exception to static query files +- Keep write inputs separate from stored rows and keep test-only assertions out of public types +- `FromRow` supports runtime reads, but production static reads still use `query_file_as!` for compile-time checking + +Database integration tests are ignored by default, but still compile in workspace checks. Run them explicitly against a local database already migrated by Prisma. Missing or invalid database configuration must fail the selected tests: + +```sh +DATABASE_URL=postgresql://... cargo test -p litellm-db --locked -- --include-ignored +``` + +After changing a query, refresh the checked query metadata from this directory: + +```sh +SQLX_OFFLINE=false cargo sqlx prepare +``` + +Tests use transactions and leave them uncommitted so fixture rows roll back when each test finishes diff --git a/litellm-rust/crates/db/Cargo.toml b/litellm-rust/crates/db/Cargo.toml index ca9d6f555b0..4f8fe4e28e2 100644 --- a/litellm-rust/crates/db/Cargo.toml +++ b/litellm-rust/crates/db/Cargo.toml @@ -6,5 +6,14 @@ license.workspace = true repository.workspace = true [dependencies] +chrono.workspace = true +macro_rules_attribute.workspace = true serde.workspace = true +serde_json.workspace = true sqlx.workspace = true +strum.workspace = true +thiserror.workspace = true + +[dev-dependencies] +rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/db/queries/end_user/find_end_user.sql b/litellm-rust/crates/db/queries/end_user/find_end_user.sql new file mode 100644 index 00000000000..c10ef8b877d --- /dev/null +++ b/litellm-rust/crates/db/queries/end_user/find_end_user.sql @@ -0,0 +1,10 @@ +SELECT e.spend AS "spend?", + e.blocked AS "blocked?", + b.max_budget AS "max_budget?", + b.rpm_limit AS "rpm_limit?", + b.tpm_limit AS "tpm_limit?" +FROM (SELECT 1) AS one +LEFT JOIN "LiteLLM_EndUserTable" AS e ON e.user_id = $1 +LEFT JOIN "LiteLLM_BudgetTable" AS assigned ON assigned.budget_id = e.budget_id +LEFT JOIN "LiteLLM_BudgetTable" AS key_budget ON key_budget.budget_id = $3 +LEFT JOIN "LiteLLM_BudgetTable" AS b ON b.budget_id = COALESCE(assigned.budget_id, key_budget.budget_id, $2) diff --git a/litellm-rust/crates/db/queries/spend/add_end_user_spend.sql b/litellm-rust/crates/db/queries/spend/add_end_user_spend.sql new file mode 100644 index 00000000000..06a5b96d254 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/add_end_user_spend.sql @@ -0,0 +1,6 @@ +INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked) +SELECT id, SUM(cost), false FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost) +GROUP BY id +ORDER BY id +ON CONFLICT (user_id) DO UPDATE +SET spend = "LiteLLM_EndUserTable".spend + EXCLUDED.spend diff --git a/litellm-rust/crates/db/queries/spend/add_key_spend.sql b/litellm-rust/crates/db/queries/spend/add_key_spend.sql new file mode 100644 index 00000000000..61850714acd --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/add_key_spend.sql @@ -0,0 +1,16 @@ +WITH added AS ( + SELECT id, SUM(cost) AS cost + FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost) + GROUP BY id +), locked AS ( + SELECT t.token FROM "LiteLLM_VerificationToken" t + WHERE t.token IN (SELECT id FROM added) + ORDER BY t.token FOR UPDATE +) +UPDATE "LiteLLM_VerificationToken" t +SET spend = t.spend + added.cost, + total_spend = t.total_spend + added.cost, + last_active = (clock_timestamp() AT TIME ZONE 'UTC'), + updated_at = (clock_timestamp() AT TIME ZONE 'UTC') +FROM added +WHERE t.token = added.id AND t.token IN (SELECT token FROM locked) diff --git a/litellm-rust/crates/db/queries/spend/add_team_spend.sql b/litellm-rust/crates/db/queries/spend/add_team_spend.sql new file mode 100644 index 00000000000..58b8e3d6ab9 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/add_team_spend.sql @@ -0,0 +1,15 @@ +WITH added AS ( + SELECT id, SUM(cost) AS cost + FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost) + GROUP BY id +), +locked AS ( + SELECT t.team_id FROM "LiteLLM_TeamTable" t + WHERE t.team_id IN (SELECT id FROM added) + ORDER BY t.team_id FOR UPDATE +) +UPDATE "LiteLLM_TeamTable" t +SET spend = t.spend + added.cost, + updated_at = (NOW() AT TIME ZONE 'UTC') +FROM added +WHERE t.team_id = added.id AND t.team_id IN (SELECT team_id FROM locked) diff --git a/litellm-rust/crates/db/queries/spend/add_user_spend.sql b/litellm-rust/crates/db/queries/spend/add_user_spend.sql new file mode 100644 index 00000000000..27fc12e82f0 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/add_user_spend.sql @@ -0,0 +1,14 @@ +WITH added AS ( + SELECT id, SUM(cost) AS cost + FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost) + GROUP BY id +), locked AS ( + SELECT u.user_id FROM "LiteLLM_UserTable" u + WHERE u.user_id IN (SELECT id FROM added) + ORDER BY u.user_id FOR UPDATE +) +UPDATE "LiteLLM_UserTable" u +SET spend = u.spend + added.cost, + updated_at = (NOW() AT TIME ZONE 'UTC') +FROM added +WHERE u.user_id = added.id AND u.user_id IN (SELECT user_id FROM locked) diff --git a/litellm-rust/crates/db/queries/spend/insert_spend_logs.sql b/litellm-rust/crates/db/queries/spend/insert_spend_logs.sql new file mode 100644 index 00000000000..fcffee181a1 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/insert_spend_logs.sql @@ -0,0 +1,26 @@ +INSERT INTO "LiteLLM_SpendLogs" ( + request_id, call_type, api_key, spend, total_tokens, prompt_tokens, + completion_tokens, "startTime", "endTime", "completionStartTime", + request_duration_ms, model, model_id, model_group, custom_llm_provider, api_base, + "user", metadata, cache_hit, cache_key, request_tags, team_id, organization_id, + end_user, requester_ip_address, messages, response, proxy_server_request, + session_id, status, litellm_call_id +) +SELECT l.request_id, l.call_type, l.api_key, l.spend, l.total_tokens, l.prompt_tokens, + l.completion_tokens, l.start_time::timestamp(3), l.end_time::timestamp(3), + l.completion_start_time::timestamp(3), l.request_duration_ms, l.model, + l.model_id, l.model_group, l.custom_llm_provider, l.api_base, l.user_id, + l.metadata, l.cache_hit, l.cache_key, '[]'::jsonb, l.team_id, + l.organization_id, l.end_user, l.requester_ip_address, '{}'::jsonb, + '{}'::jsonb, '{}'::jsonb, l.session_id, l.status, l.litellm_call_id +FROM UNNEST($1::text[], $2::text[], $3::text[], $4::float8[], $5::int4[], $6::int4[], + $7::int4[], $8::timestamp[], $9::timestamp[], $10::timestamp[], $11::int4[], $12::text[], + $13::text[], $14::text[], $15::text[], $16::text[], $17::text[], + $18::jsonb[], $19::text[], $20::text[], $21::text[], $22::text[], + $23::text[], $24::text[], $25::text[], $26::text[], $27::text[]) + AS l(request_id, call_type, api_key, spend, total_tokens, prompt_tokens, + completion_tokens, start_time, end_time, completion_start_time, + request_duration_ms, model, model_id, model_group, custom_llm_provider, + api_base, user_id, metadata, cache_hit, cache_key, team_id, organization_id, + end_user, requester_ip_address, session_id, status, litellm_call_id) +ON CONFLICT DO NOTHING diff --git a/litellm-rust/crates/db/queries/spend/lock_team.sql b/litellm-rust/crates/db/queries/spend/lock_team.sql new file mode 100644 index 00000000000..06cdb806ef0 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/lock_team.sql @@ -0,0 +1 @@ +SELECT pg_advisory_xact_lock(hashtext($1)) IS NULL AS "locked!" diff --git a/litellm-rust/crates/db/queries/spend/upsert_daily_end_user_spend.sql b/litellm-rust/crates/db/queries/spend/upsert_daily_end_user_spend.sql new file mode 100644 index 00000000000..8c4936e1a7a --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/upsert_daily_end_user_spend.sql @@ -0,0 +1,40 @@ +INSERT INTO "LiteLLM_DailyEndUserSpend" ( + id, end_user_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens, + completion_tokens, api_requests, successful_requests, failed_requests, + cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms, + timed_requests, spend, updated_at +) +SELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model, + r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens, + r.completion_tokens, r.api_requests, r.successful_requests, + r.failed_requests, r.cache_read, r.cache_creation, r.response_time, + r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC') +FROM UNNEST( + $1::text[], $2::text[], $3::text[], $4::text[], $5::text[], + $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[], + $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[], + $16::int8[], $17::float8[], $18::text[] +) AS r( + entity_id, date, api_key, model, custom_llm_provider, endpoint, + model_group, prompt_tokens, completion_tokens, api_requests, + successful_requests, failed_requests, cache_read, cache_creation, + response_time, timed_requests, spend, mcp_namespaced_tool_name +) +ORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider, + r.mcp_namespaced_tool_name, r.endpoint +ON CONFLICT ( + end_user_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint +) DO UPDATE SET + prompt_tokens = "LiteLLM_DailyEndUserSpend".prompt_tokens + EXCLUDED.prompt_tokens, + completion_tokens = "LiteLLM_DailyEndUserSpend".completion_tokens + EXCLUDED.completion_tokens, + api_requests = "LiteLLM_DailyEndUserSpend".api_requests + EXCLUDED.api_requests, + successful_requests = "LiteLLM_DailyEndUserSpend".successful_requests + EXCLUDED.successful_requests, + failed_requests = "LiteLLM_DailyEndUserSpend".failed_requests + EXCLUDED.failed_requests, + cache_read_input_tokens = "LiteLLM_DailyEndUserSpend".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens, + cache_creation_input_tokens = "LiteLLM_DailyEndUserSpend".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens, + total_response_time_ms = "LiteLLM_DailyEndUserSpend".total_response_time_ms + EXCLUDED.total_response_time_ms, + timed_requests = "LiteLLM_DailyEndUserSpend".timed_requests + EXCLUDED.timed_requests, + spend = "LiteLLM_DailyEndUserSpend".spend + EXCLUDED.spend, + updated_at = (NOW() AT TIME ZONE 'UTC') diff --git a/litellm-rust/crates/db/queries/spend/upsert_daily_team_spend.sql b/litellm-rust/crates/db/queries/spend/upsert_daily_team_spend.sql new file mode 100644 index 00000000000..a9bdef34043 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/upsert_daily_team_spend.sql @@ -0,0 +1,40 @@ +INSERT INTO "LiteLLM_DailyTeamSpend" ( + id, team_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens, + completion_tokens, api_requests, successful_requests, failed_requests, + cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms, + timed_requests, spend, updated_at +) +SELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model, + r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens, + r.completion_tokens, r.api_requests, r.successful_requests, + r.failed_requests, r.cache_read, r.cache_creation, r.response_time, + r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC') +FROM UNNEST( + $1::text[], $2::text[], $3::text[], $4::text[], $5::text[], + $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[], + $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[], + $16::int8[], $17::float8[], $18::text[] +) AS r( + entity_id, date, api_key, model, custom_llm_provider, endpoint, + model_group, prompt_tokens, completion_tokens, api_requests, + successful_requests, failed_requests, cache_read, cache_creation, + response_time, timed_requests, spend, mcp_namespaced_tool_name +) +ORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider, + r.mcp_namespaced_tool_name, r.endpoint +ON CONFLICT ( + team_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint +) DO UPDATE SET + prompt_tokens = "LiteLLM_DailyTeamSpend".prompt_tokens + EXCLUDED.prompt_tokens, + completion_tokens = "LiteLLM_DailyTeamSpend".completion_tokens + EXCLUDED.completion_tokens, + api_requests = "LiteLLM_DailyTeamSpend".api_requests + EXCLUDED.api_requests, + successful_requests = "LiteLLM_DailyTeamSpend".successful_requests + EXCLUDED.successful_requests, + failed_requests = "LiteLLM_DailyTeamSpend".failed_requests + EXCLUDED.failed_requests, + cache_read_input_tokens = "LiteLLM_DailyTeamSpend".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens, + cache_creation_input_tokens = "LiteLLM_DailyTeamSpend".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens, + total_response_time_ms = "LiteLLM_DailyTeamSpend".total_response_time_ms + EXCLUDED.total_response_time_ms, + timed_requests = "LiteLLM_DailyTeamSpend".timed_requests + EXCLUDED.timed_requests, + spend = "LiteLLM_DailyTeamSpend".spend + EXCLUDED.spend, + updated_at = (NOW() AT TIME ZONE 'UTC') diff --git a/litellm-rust/crates/db/queries/spend/upsert_daily_user_spend.sql b/litellm-rust/crates/db/queries/spend/upsert_daily_user_spend.sql new file mode 100644 index 00000000000..4a5db90dee2 --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/upsert_daily_user_spend.sql @@ -0,0 +1,40 @@ +INSERT INTO "LiteLLM_DailyUserSpend" ( + id, user_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens, + completion_tokens, api_requests, successful_requests, failed_requests, + cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms, + timed_requests, spend, updated_at +) +SELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model, + r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens, + r.completion_tokens, r.api_requests, r.successful_requests, + r.failed_requests, r.cache_read, r.cache_creation, r.response_time, + r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC') +FROM UNNEST( + $1::text[], $2::text[], $3::text[], $4::text[], $5::text[], + $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[], + $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[], + $16::int8[], $17::float8[], $18::text[] +) AS r( + entity_id, date, api_key, model, custom_llm_provider, endpoint, + model_group, prompt_tokens, completion_tokens, api_requests, + successful_requests, failed_requests, cache_read, cache_creation, + response_time, timed_requests, spend, mcp_namespaced_tool_name +) +ORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider, + r.mcp_namespaced_tool_name, r.endpoint +ON CONFLICT ( + user_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint +) DO UPDATE SET + prompt_tokens = "LiteLLM_DailyUserSpend".prompt_tokens + EXCLUDED.prompt_tokens, + completion_tokens = "LiteLLM_DailyUserSpend".completion_tokens + EXCLUDED.completion_tokens, + api_requests = "LiteLLM_DailyUserSpend".api_requests + EXCLUDED.api_requests, + successful_requests = "LiteLLM_DailyUserSpend".successful_requests + EXCLUDED.successful_requests, + failed_requests = "LiteLLM_DailyUserSpend".failed_requests + EXCLUDED.failed_requests, + cache_read_input_tokens = "LiteLLM_DailyUserSpend".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens, + cache_creation_input_tokens = "LiteLLM_DailyUserSpend".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens, + total_response_time_ms = "LiteLLM_DailyUserSpend".total_response_time_ms + EXCLUDED.total_response_time_ms, + timed_requests = "LiteLLM_DailyUserSpend".timed_requests + EXCLUDED.timed_requests, + spend = "LiteLLM_DailyUserSpend".spend + EXCLUDED.spend, + updated_at = (NOW() AT TIME ZONE 'UTC') diff --git a/litellm-rust/crates/db/queries/spend/upsert_team_member_spend.sql b/litellm-rust/crates/db/queries/spend/upsert_team_member_spend.sql new file mode 100644 index 00000000000..460c631cbbd --- /dev/null +++ b/litellm-rust/crates/db/queries/spend/upsert_team_member_spend.sql @@ -0,0 +1,15 @@ +INSERT INTO "LiteLLM_TeamMembership" (user_id, team_id, spend, total_spend) +SELECT member.user_id, member.team_id, member.cost, member.cost +FROM jsonb_to_recordset($1::jsonb) AS member(user_id text, team_id text, cost float8) +WHERE EXISTS ( + SELECT 1 FROM "LiteLLM_TeamTable" t + WHERE t.team_id = member.team_id + AND t.members_with_roles @> jsonb_build_array(jsonb_build_object('user_id', member.user_id)) +) + OR EXISTS ( + SELECT 1 FROM "LiteLLM_TeamMembership" m + WHERE m.user_id = member.user_id AND m.team_id = member.team_id +) +ON CONFLICT (user_id, team_id) DO UPDATE +SET spend = "LiteLLM_TeamMembership".spend + EXCLUDED.spend, + total_spend = "LiteLLM_TeamMembership".total_spend + EXCLUDED.total_spend diff --git a/litellm-rust/crates/db/src/daily_spend.rs b/litellm-rust/crates/db/src/daily_spend.rs new file mode 100644 index 00000000000..9db07514aab --- /dev/null +++ b/litellm-rust/crates/db/src/daily_spend.rs @@ -0,0 +1,76 @@ +use sqlx::types::chrono::NaiveDateTime; + +#[macro_rules_attribute::apply(crate::db_row)] +pub struct DailyUserRow { + pub id: String, + pub user_id: Option, + pub api_key: String, + pub model: Option, + pub custom_llm_provider: Option, + pub endpoint: Option, + pub model_group: Option, + pub mcp_namespaced_tool_name: Option, + pub prompt_tokens: i64, + pub completion_tokens: i64, + pub api_requests: i64, + pub successful_requests: i64, + pub failed_requests: i64, + pub cache_read_input_tokens: i64, + pub cache_creation_input_tokens: i64, + pub total_response_time_ms: i64, + pub timed_requests: i64, + pub spend: f64, + pub date: String, + pub created_at: NaiveDateTime, + pub updated_at: NaiveDateTime, +} + +#[macro_rules_attribute::apply(crate::db_row)] +pub struct DailyTeamRow { + pub id: String, + pub team_id: Option, + pub api_key: String, + pub model: Option, + pub custom_llm_provider: Option, + pub endpoint: Option, + pub model_group: Option, + pub mcp_namespaced_tool_name: Option, + pub prompt_tokens: i64, + pub completion_tokens: i64, + pub api_requests: i64, + pub successful_requests: i64, + pub failed_requests: i64, + pub cache_read_input_tokens: i64, + pub cache_creation_input_tokens: i64, + pub total_response_time_ms: i64, + pub timed_requests: i64, + pub spend: f64, + pub date: String, + pub created_at: NaiveDateTime, + pub updated_at: NaiveDateTime, +} + +#[macro_rules_attribute::apply(crate::db_row)] +pub struct DailyEndUserRow { + pub id: String, + pub end_user_id: Option, + pub api_key: String, + pub model: Option, + pub custom_llm_provider: Option, + pub endpoint: Option, + pub model_group: Option, + pub mcp_namespaced_tool_name: Option, + pub prompt_tokens: i64, + pub completion_tokens: i64, + pub api_requests: i64, + pub successful_requests: i64, + pub failed_requests: i64, + pub cache_read_input_tokens: i64, + pub cache_creation_input_tokens: i64, + pub total_response_time_ms: i64, + pub timed_requests: i64, + pub spend: f64, + pub date: String, + pub created_at: NaiveDateTime, + pub updated_at: NaiveDateTime, +} diff --git a/litellm-rust/crates/db/src/end_user.rs b/litellm-rust/crates/db/src/end_user.rs new file mode 100644 index 00000000000..66661b51640 --- /dev/null +++ b/litellm-rust/crates/db/src/end_user.rs @@ -0,0 +1,359 @@ +use sqlx::PgExecutor; + +use crate::Error; + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct RateLimits { + pub rpm: Option, + pub tpm: Option, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct EndUser { + pub spend: Option, + pub blocked: bool, + pub max_budget: Option, + pub limits: RateLimits, +} + +#[macro_rules_attribute::apply(crate::db_row)] +struct EndUserRow { + spend: Option, + blocked: Option, + max_budget: Option, + rpm_limit: Option, + tpm_limit: Option, +} + +pub async fn find_end_user( + executor: impl PgExecutor<'_>, + end_user_id: &str, + default_budget_id: Option<&str>, + key_budget_id: Option<&str>, +) -> Result { + let row = sqlx::query_file_as!( + EndUserRow, + "queries/end_user/find_end_user.sql", + end_user_id, + default_budget_id, + key_budget_id, + ) + .fetch_one(executor) + .await?; + Ok(EndUser { + spend: row.spend, + blocked: row.blocked.unwrap_or(false), + max_budget: row.max_budget, + limits: RateLimits { + rpm: row.rpm_limit, + tpm: row.tpm_limit, + }, + }) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{EndUser, RateLimits, find_end_user}; + use crate::test_support::database_pool; + use rstest::rstest; + use sqlx::{PgPool, Postgres, Transaction}; + + #[derive(Clone, Copy)] + struct BudgetFixture { + budget_id: &'static str, + max_budget: f64, + rpm_limit: i64, + tpm_limit: i64, + } + + async fn insert_budget( + tx: &mut Transaction<'_, Postgres>, + budget: BudgetFixture, + ) -> Result<(), sqlx::Error> { + #[expect( + clippy::disallowed_methods, + reason = "Fixture inserts are outside the compile-checked query set" + )] + let result = sqlx::query( + r#" + INSERT INTO "LiteLLM_BudgetTable" + (budget_id, max_budget, rpm_limit, tpm_limit, created_by, updated_by) + VALUES ($1, $2, $3, $4, 'db-query-test', 'db-query-test') + "#, + ) + .bind(budget.budget_id) + .bind(budget.max_budget) + .bind(budget.rpm_limit) + .bind(budget.tpm_limit) + .execute(&mut **tx) + .await; + result.map(|_| ()) + } + + async fn insert_end_user( + tx: &mut Transaction<'_, Postgres>, + user_id: &str, + spend: f64, + blocked: bool, + budget_id: Option<&str>, + ) -> Result<(), sqlx::Error> { + #[expect( + clippy::disallowed_methods, + reason = "Fixture inserts are outside the compile-checked query set" + )] + let result = sqlx::query( + r#" + INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked, budget_id) + VALUES ($1, $2, $3, $4) + "#, + ) + .bind(user_id) + .bind(spend) + .bind(blocked) + .bind(budget_id) + .execute(&mut **tx) + .await; + result.map(|_| ()) + } + + #[rstest] + #[case::without_default_budget(None)] + #[case::with_default_budget(Some(BudgetFixture { + budget_id: "missing-user-default-budget", + max_budget: 11.5, + rpm_limit: 12, + tpm_limit: 1300, + }))] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn missing_end_user_maps_to_defaults( + #[case] default_budget: Option, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + if let Some(budget) = default_budget { + insert_budget(&mut tx, budget).await?; + } + + let actual = find_end_user( + &mut *tx, + "missing-end-user", + default_budget.map(|budget| budget.budget_id), + None, + ) + .await?; + + assert_eq!( + actual, + EndUser { + spend: None, + blocked: false, + max_budget: default_budget.map(|budget| budget.max_budget), + limits: RateLimits { + rpm: default_budget.map(|budget| budget.rpm_limit), + tpm: default_budget.map(|budget| budget.tpm_limit), + }, + } + ); + Ok(()) + } + + #[rstest] + #[case::row_budget_precedes_default( + "row-budget-priority-user", + Some(BudgetFixture { + budget_id: "row-budget-priority-row", + max_budget: 10.0, + rpm_limit: 101, + tpm_limit: 1001, + }), + BudgetFixture { + budget_id: "row-budget-priority-default", + max_budget: 20.0, + rpm_limit: 201, + tpm_limit: 2001, + }, + None, + Some(BudgetFixture { + budget_id: "row-budget-priority-row", + max_budget: 10.0, + rpm_limit: 101, + tpm_limit: 1001, + }), + false + )] + #[case::default_applies_without_row_budget( + "default-budget-user", + None, + BudgetFixture { + budget_id: "default-budget-fallback", + max_budget: 30.0, + rpm_limit: 301, + tpm_limit: 3001, + }, + None, + Some(BudgetFixture { + budget_id: "default-budget-fallback", + max_budget: 30.0, + rpm_limit: 301, + tpm_limit: 3001, + }), + false + )] + #[case::assigned_budget_precedes_key_when_all_ids_differ( + "override-priority-user", + Some(BudgetFixture { + budget_id: "override-priority-row", + max_budget: 40.0, + rpm_limit: 401, + tpm_limit: 4001, + }), + BudgetFixture { + budget_id: "override-priority-default", + max_budget: 50.0, + rpm_limit: 501, + tpm_limit: 5001, + }, + Some(BudgetFixture { + budget_id: "override-priority-override", + max_budget: 60.0, + rpm_limit: 601, + tpm_limit: 6001, + }), + Some(BudgetFixture { + budget_id: "override-priority-row", + max_budget: 40.0, + rpm_limit: 401, + tpm_limit: 4001, + }), + true + )] + #[case::missing_key_budget_keeps_assigned_budget( + "dangling-override-user", + Some(BudgetFixture { + budget_id: "dangling-override-row", + max_budget: 70.0, + rpm_limit: 701, + tpm_limit: 7001, + }), + BudgetFixture { + budget_id: "dangling-override-default", + max_budget: 80.0, + rpm_limit: 801, + tpm_limit: 8001, + }, + Some(BudgetFixture { + budget_id: "dangling-override-missing", + max_budget: 90.0, + rpm_limit: 901, + tpm_limit: 9001, + }), + Some(BudgetFixture { + budget_id: "dangling-override-row", + max_budget: 70.0, + rpm_limit: 701, + tpm_limit: 7001, + }), + false + )] + #[case::key_budget_precedes_proxy_default( + "key-default-user", + None, + BudgetFixture { + budget_id: "key-default-proxy", + max_budget: 80.0, + rpm_limit: 801, + tpm_limit: 8001, + }, + Some(BudgetFixture { + budget_id: "key-default-key", + max_budget: 90.0, + rpm_limit: 901, + tpm_limit: 9001, + }), + Some(BudgetFixture { + budget_id: "key-default-key", + max_budget: 90.0, + rpm_limit: 901, + tpm_limit: 9001, + }), + true + )] + #[case::missing_key_budget_falls_back_to_proxy_default( + "missing-key-default-user", + None, + BudgetFixture { + budget_id: "missing-key-default-proxy", + max_budget: 80.0, + rpm_limit: 801, + tpm_limit: 8001, + }, + Some(BudgetFixture { + budget_id: "missing-key-default-key", + max_budget: 90.0, + rpm_limit: 901, + tpm_limit: 9001, + }), + Some(BudgetFixture { + budget_id: "missing-key-default-proxy", + max_budget: 80.0, + rpm_limit: 801, + tpm_limit: 8001, + }), + false + )] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn budget_resolution_matches_python_precedence( + #[case] user_id: &'static str, + #[case] user_budget: Option, + #[case] default_budget: BudgetFixture, + #[case] override_budget: Option, + #[case] expected_budget: Option, + #[case] insert_override_budget: bool, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + insert_budget(&mut tx, default_budget).await?; + if let Some(budget) = user_budget { + insert_budget(&mut tx, budget).await?; + } + if let (Some(budget), true) = (override_budget, insert_override_budget) { + insert_budget(&mut tx, budget).await?; + } + insert_end_user( + &mut tx, + user_id, + 7.5, + true, + user_budget.map(|budget| budget.budget_id), + ) + .await?; + + let actual = find_end_user( + &mut *tx, + user_id, + Some(default_budget.budget_id), + override_budget.map(|budget| budget.budget_id), + ) + .await?; + + assert_eq!( + actual, + EndUser { + spend: Some(7.5), + blocked: true, + max_budget: expected_budget.map(|budget| budget.max_budget), + limits: RateLimits { + rpm: expected_budget.map(|budget| budget.rpm_limit), + tpm: expected_budget.map(|budget| budget.tpm_limit), + }, + } + ); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/error.rs b/litellm-rust/crates/db/src/error.rs new file mode 100644 index 00000000000..af0ee030896 --- /dev/null +++ b/litellm-rust/crates/db/src/error.rs @@ -0,0 +1,79 @@ +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("database query failed")] + Query(#[from] sqlx::Error), + #[error("spend batch has {ids} ids and {costs} costs")] + InvalidSpendBatch { ids: usize, costs: usize }, + #[error("invalid spend timestamp")] + InvalidTimestamp(#[from] chrono::ParseError), +} + +impl Error { + pub fn is_retryable(&self) -> bool { + match self { + Self::Query(sqlx::Error::Database(error)) => { + matches!( + error.code().as_deref(), + Some("40P01" | "55P03" | "40001" | "53300") + ) + } + Self::Query(sqlx::Error::Io(_) | sqlx::Error::PoolTimedOut) => true, + _ => false, + } + } +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use rstest::rstest; + use sqlx::PgPool; + + use super::Error; + use crate::test_support::database_pool; + + #[rstest] + #[case::closed_pool(sqlx::Error::PoolClosed, false)] + #[case::pool_timeout(sqlx::Error::PoolTimedOut, true)] + #[case::io(sqlx::Error::Io(std::io::ErrorKind::ConnectionReset.into()), true)] + #[case::missing_row(sqlx::Error::RowNotFound, false)] + fn classifies_client_errors(#[case] error: sqlx::Error, #[case] retryable: bool) { + assert_eq!(Error::from(error).is_retryable(), retryable); + } + + #[rstest] + #[case::connection_capacity("53300", true)] + #[case::deadlock("40P01", true)] + #[case::lock_unavailable("55P03", true)] + #[case::serialization_failure("40001", true)] + #[case::unique_violation("23505", false)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn classifies_server_errors( + #[case] code: &str, + #[case] retryable: bool, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test query must raise the requested SQLSTATE" + )] + let error = sqlx::query(sqlx::AssertSqlSafe(format!( + "DO $$ BEGIN RAISE EXCEPTION USING ERRCODE = '{code}'; END $$" + ))) + .execute(&mut *tx) + .await + .expect_err("PostgreSQL must raise the requested error"); + + assert_eq!( + error.as_database_error().and_then(|error| error.code()), + Some(code.into()) + ); + assert_eq!(Error::from(error).is_retryable(), retryable); + tx.rollback().await?; + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/lib.rs b/litellm-rust/crates/db/src/lib.rs index b7ce211d906..541f97c7db9 100644 --- a/litellm-rust/crates/db/src/lib.rs +++ b/litellm-rust/crates/db/src/lib.rs @@ -1,3 +1,26 @@ -mod failure; +macro_rules_attribute::attribute_alias! { + #[apply(db_input)] = + #[derive(Clone, Debug, PartialEq)]; + #[apply(db_row)] = + #[derive(Clone, Debug, PartialEq, sqlx::FromRow)]; +} + +mod daily_spend; +mod end_user; +mod error; +mod failure; +mod spend; +#[cfg(test)] +mod test_support; + +pub use daily_spend::{DailyEndUserRow, DailyTeamRow, DailyUserRow}; +pub use end_user::{EndUser, RateLimits, find_end_user}; +pub use error::Error; pub use failure::DbFailure; +pub use spend::{ + DailySpend, DailyTable, EntitySpend, SpendLog, SpendLogRow, TeamMemberSpend, + add_end_user_spend, add_key_spend, add_team_member_spend, add_team_spend, add_user_spend, + begin, insert_spend_logs, lock_team, upsert_daily_end_user_spend, upsert_daily_spend, + upsert_daily_team_spend, upsert_daily_user_spend, +}; diff --git a/litellm-rust/crates/db/src/spend/daily_end_user_spend.rs b/litellm-rust/crates/db/src/spend/daily_end_user_spend.rs new file mode 100644 index 00000000000..6c88d47cdcc --- /dev/null +++ b/litellm-rust/crates/db/src/spend/daily_end_user_spend.rs @@ -0,0 +1,584 @@ +use sqlx::PgExecutor; + +use super::{DailySpend, upsert_daily_team_spend, upsert_daily_user_spend}; +use crate::Error; + +pub async fn upsert_daily_end_user_spend( + executor: impl PgExecutor<'_>, + rows: &[DailySpend], +) -> Result<(), Error> { + if rows.is_empty() { + return Ok(()); + } + macro_rules! column { + ($field:ident) => { + rows.iter() + .map(|row| row.$field.clone()) + .collect::>() + }; + } + sqlx::query_file!( + "queries/spend/upsert_daily_end_user_spend.sql", + &column!(entity_id), + &column!(date), + &column!(api_key), + &column!(model), + &column!(custom_llm_provider), + &column!(endpoint), + &column!(model_group) as &[Option], + &column!(prompt_tokens), + &column!(completion_tokens), + &column!(api_requests), + &column!(successful_requests), + &column!(failed_requests), + &column!(cache_read_input_tokens), + &column!(cache_creation_input_tokens), + &column!(total_response_time_ms), + &column!(timed_requests), + &column!(spend), + &column!(mcp_namespaced_tool_name), + ) + .execute(executor) + .await?; + Ok(()) +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, strum::VariantArray)] +pub enum DailyTable { + User, + Team, + EndUser, +} + +pub async fn upsert_daily_spend( + executor: impl PgExecutor<'_>, + table: DailyTable, + rows: &[DailySpend], +) -> Result<(), Error> { + match table { + DailyTable::User => upsert_daily_user_spend(executor, rows).await, + DailyTable::Team => upsert_daily_team_spend(executor, rows).await, + DailyTable::EndUser => upsert_daily_end_user_spend(executor, rows).await, + } +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{DailySpend, DailyTable, upsert_daily_end_user_spend, upsert_daily_spend}; + use crate::{DailyEndUserRow, test_support::database_pool}; + use rstest::rstest; + use sqlx::{PgPool, types::chrono::NaiveDateTime}; + + #[rstest] + #[case::full_keys( + "daily-end-user-base", + "daily-key-base", + "daily-model", + "daily-provider", + "/chat/completions", + Some("first-group") + )] + #[case::initial_model_group_none( + "daily-end-user-base", + "daily-key-base", + "daily-model", + "daily-provider", + "/chat/completions", + None + )] + #[case::empty_optional_keys("", "", "", "", "", Some(""))] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_end_user_spend_upsert_preserves_keys_and_accumulates( + #[case] end_user_id: &'static str, + #[case] api_key: &'static str, + #[case] model: &'static str, + #[case] custom_llm_provider: &'static str, + #[case] endpoint: &'static str, + #[case] first_model_group: Option<&'static str>, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test exercises UTC timestamp behavior under a non-UTC session" + )] + sqlx::query("SET LOCAL TIME ZONE 'America/Los_Angeles'") + .execute(&mut *tx) + .await?; + + let first_batch = [ + DailySpend { + entity_id: end_user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: first_model_group.map(str::to_owned), + prompt_tokens: 1, + completion_tokens: 2, + api_requests: 3, + successful_requests: 4, + failed_requests: 5, + cache_read_input_tokens: 6, + cache_creation_input_tokens: 7, + total_response_time_ms: 8, + timed_requests: 9, + spend: 0.125, + }, + DailySpend { + entity_id: end_user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: "endpoint-sentinel".to_owned(), + model_group: Some("sentinel-endpoint-group".to_owned()), + prompt_tokens: 101, + completion_tokens: 102, + api_requests: 103, + successful_requests: 104, + failed_requests: 105, + cache_read_input_tokens: 106, + cache_creation_input_tokens: 107, + total_response_time_ms: 108, + timed_requests: 109, + spend: 1.25, + }, + DailySpend { + entity_id: end_user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: "api-key-sentinel".to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("sentinel-key-group".to_owned()), + prompt_tokens: 201, + completion_tokens: 202, + api_requests: 203, + successful_requests: 204, + failed_requests: 205, + cache_read_input_tokens: 206, + cache_creation_input_tokens: 207, + total_response_time_ms: 208, + timed_requests: 209, + spend: 2.5, + }, + DailySpend { + entity_id: "entity-sentinel".to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("sentinel-entity-group".to_owned()), + prompt_tokens: 301, + completion_tokens: 302, + api_requests: 303, + successful_requests: 304, + failed_requests: 305, + cache_read_input_tokens: 306, + cache_creation_input_tokens: 307, + total_response_time_ms: 308, + timed_requests: 309, + spend: 3.75, + }, + ]; + upsert_daily_end_user_spend(&mut *tx, &first_batch).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test captures the inserted row identity before conflict handling" + )] + let (initial_id, initial_created_at): (String, NaiveDateTime) = sqlx::query_as( + r#"SELECT id, created_at + FROM "LiteLLM_DailyEndUserSpend" + WHERE end_user_id = $1 AND date = $2 AND api_key = $3 AND model = $4 + AND custom_llm_provider = $5 AND endpoint = $6"#, + ) + .bind(end_user_id) + .bind("2026-01-15") + .bind(api_key) + .bind(model) + .bind(custom_llm_provider) + .bind(endpoint) + .fetch_one(&mut *tx) + .await?; + assert!(!initial_id.is_empty()); + + #[expect( + clippy::disallowed_methods, + reason = "The integration test seeds updated_at before exercising the conflict update" + )] + let seeded_updated_at: NaiveDateTime = sqlx::query_scalar( + r#"UPDATE "LiteLLM_DailyEndUserSpend" + SET updated_at = TIMESTAMP '2000-01-01' + WHERE id = $1 + RETURNING updated_at"#, + ) + .bind(&initial_id) + .fetch_one(&mut *tx) + .await?; + + let increment = [DailySpend { + entity_id: end_user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("second-group".to_owned()), + prompt_tokens: 10, + completion_tokens: 20, + api_requests: 30, + successful_requests: 40, + failed_requests: 50, + cache_read_input_tokens: 60, + cache_creation_input_tokens: 70, + total_response_time_ms: 80, + timed_requests: 90, + spend: 0.625, + }]; + upsert_daily_end_user_spend(&mut *tx, &increment).await?; + upsert_daily_end_user_spend(&mut *tx, &[]).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test captures the transaction timestamp for assertions" + )] + let expected_updated_at: NaiveDateTime = + sqlx::query_scalar("SELECT (NOW() AT TIME ZONE 'UTC')::timestamp(3)") + .fetch_one(&mut *tx) + .await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test reads all rows to verify attribution and accumulation" + )] + let rows: Vec = sqlx::query_as( + r#"SELECT id, end_user_id, api_key, model, custom_llm_provider, endpoint, + model_group, mcp_namespaced_tool_name, prompt_tokens, + completion_tokens, api_requests, successful_requests, + failed_requests, cache_read_input_tokens, + cache_creation_input_tokens, total_response_time_ms, + timed_requests, spend, date, created_at, updated_at + FROM "LiteLLM_DailyEndUserSpend" + WHERE date = $1 AND model = $2 AND custom_llm_provider = $3 + AND mcp_namespaced_tool_name = '' AND end_user_id = ANY($4) + ORDER BY CASE + WHEN end_user_id = $5 AND api_key = $6 AND endpoint = $7 THEN 0 + WHEN end_user_id = $5 AND api_key = $6 AND endpoint = $9 THEN 1 + WHEN end_user_id = $5 AND api_key = $8 AND endpoint = $7 THEN 2 + WHEN end_user_id = $10 AND api_key = $6 AND endpoint = $7 THEN 3 + ELSE 4 + END"#, + ) + .bind("2026-01-15") + .bind(model) + .bind(custom_llm_provider) + .bind(vec![end_user_id, "entity-sentinel"]) + .bind(end_user_id) + .bind(api_key) + .bind(endpoint) + .bind("api-key-sentinel") + .bind("endpoint-sentinel") + .bind("entity-sentinel") + .fetch_all(&mut *tx) + .await?; + + assert_eq!(rows.len(), 4); + assert_eq!(rows[0].id, initial_id); + assert_eq!(rows[0].end_user_id.as_deref(), Some(end_user_id)); + assert_eq!(rows[0].api_key, api_key); + assert_eq!(rows[0].model.as_deref(), Some(model)); + assert_eq!( + rows[0].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[0].endpoint.as_deref(), Some(endpoint)); + assert_eq!(rows[0].model_group.as_deref(), first_model_group); + assert_eq!(rows[0].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[0].prompt_tokens, + rows[0].completion_tokens, + rows[0].api_requests, + rows[0].successful_requests, + rows[0].failed_requests, + rows[0].cache_read_input_tokens, + rows[0].cache_creation_input_tokens, + rows[0].total_response_time_ms, + rows[0].timed_requests, + rows[0].spend + ), + (11, 22, 33, 44, 55, 66, 77, 88, 99, 0.75) + ); + assert_eq!(rows[0].date, "2026-01-15"); + assert_eq!(rows[0].created_at, initial_created_at); + assert!(rows[0].updated_at > seeded_updated_at); + assert_eq!(rows[0].updated_at, expected_updated_at); + + assert!(!rows[1].id.is_empty()); + assert_eq!(rows[1].end_user_id.as_deref(), Some(end_user_id)); + assert_eq!(rows[1].api_key, api_key); + assert_eq!(rows[1].model.as_deref(), Some(model)); + assert_eq!( + rows[1].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[1].endpoint.as_deref(), Some("endpoint-sentinel")); + assert_eq!( + rows[1].model_group.as_deref(), + Some("sentinel-endpoint-group") + ); + assert_eq!(rows[1].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[1].prompt_tokens, + rows[1].completion_tokens, + rows[1].api_requests, + rows[1].successful_requests, + rows[1].failed_requests, + rows[1].cache_read_input_tokens, + rows[1].cache_creation_input_tokens, + rows[1].total_response_time_ms, + rows[1].timed_requests, + rows[1].spend + ), + (101, 102, 103, 104, 105, 106, 107, 108, 109, 1.25) + ); + assert_eq!(rows[1].date, "2026-01-15"); + assert!(rows[1].updated_at > seeded_updated_at); + assert_eq!(rows[1].updated_at, expected_updated_at); + + assert!(!rows[2].id.is_empty()); + assert_eq!(rows[2].end_user_id.as_deref(), Some(end_user_id)); + assert_eq!(rows[2].api_key, "api-key-sentinel"); + assert_eq!(rows[2].model.as_deref(), Some(model)); + assert_eq!( + rows[2].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[2].endpoint.as_deref(), Some(endpoint)); + assert_eq!(rows[2].model_group.as_deref(), Some("sentinel-key-group")); + assert_eq!(rows[2].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[2].prompt_tokens, + rows[2].completion_tokens, + rows[2].api_requests, + rows[2].successful_requests, + rows[2].failed_requests, + rows[2].cache_read_input_tokens, + rows[2].cache_creation_input_tokens, + rows[2].total_response_time_ms, + rows[2].timed_requests, + rows[2].spend + ), + (201, 202, 203, 204, 205, 206, 207, 208, 209, 2.5) + ); + assert_eq!(rows[2].date, "2026-01-15"); + assert!(rows[2].updated_at > seeded_updated_at); + assert_eq!(rows[2].updated_at, expected_updated_at); + + assert!(!rows[3].id.is_empty()); + assert_eq!(rows[3].end_user_id.as_deref(), Some("entity-sentinel")); + assert_eq!(rows[3].api_key, api_key); + assert_eq!(rows[3].model.as_deref(), Some(model)); + assert_eq!( + rows[3].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[3].endpoint.as_deref(), Some(endpoint)); + assert_eq!( + rows[3].model_group.as_deref(), + Some("sentinel-entity-group") + ); + assert_eq!(rows[3].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[3].prompt_tokens, + rows[3].completion_tokens, + rows[3].api_requests, + rows[3].successful_requests, + rows[3].failed_requests, + rows[3].cache_read_input_tokens, + rows[3].cache_creation_input_tokens, + rows[3].total_response_time_ms, + rows[3].timed_requests, + rows[3].spend + ), + (301, 302, 303, 304, 305, 306, 307, 308, 309, 3.75) + ); + assert_eq!(rows[3].date, "2026-01-15"); + assert!(rows[3].updated_at > seeded_updated_at); + assert_eq!(rows[3].updated_at, expected_updated_at); + Ok(()) + } + + #[rstest] + #[case::distinct_tools("server/tool-a", "server/tool-b")] + #[case::empty_and_named_tool("", "server/tool-b")] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_end_user_spend_preserves_tool_attribution( + #[case] first_tool: &str, + #[case] second_tool: &str, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + let first = DailySpend { + entity_id: "daily-end-user-tools".to_owned(), + date: "2026-02-21".to_owned(), + api_key: "tool-key".to_owned(), + model: "tool-model".to_owned(), + custom_llm_provider: "tool-provider".to_owned(), + endpoint: "/mcp".to_owned(), + mcp_namespaced_tool_name: first_tool.to_owned(), + prompt_tokens: 2, + spend: 0.25, + ..Default::default() + }; + let second = DailySpend { + mcp_namespaced_tool_name: second_tool.to_owned(), + prompt_tokens: 5, + spend: 0.5, + ..first.clone() + }; + upsert_daily_end_user_spend(&mut *tx, &[second, first.clone()]).await?; + upsert_daily_end_user_spend(&mut *tx, &[first]).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test reads stored tool attribution and counters" + )] + let rows: Vec = sqlx::query_as( + r#"SELECT * FROM "LiteLLM_DailyEndUserSpend" + WHERE end_user_id = $1 AND date = $2 AND api_key = $3 + ORDER BY mcp_namespaced_tool_name"#, + ) + .bind("daily-end-user-tools") + .bind("2026-02-21") + .bind("tool-key") + .fetch_all(&mut *tx) + .await?; + + assert_eq!(rows.len(), 2); + assert_eq!( + rows[0].mcp_namespaced_tool_name.as_deref(), + Some(first_tool) + ); + assert_eq!( + rows[1].mcp_namespaced_tool_name.as_deref(), + Some(second_tool) + ); + assert_eq!((rows[0].prompt_tokens, rows[0].spend), (4, 0.5)); + assert_eq!((rows[1].prompt_tokens, rows[1].spend), (5, 0.5)); + assert_ne!(rows[0].id, rows[1].id); + Ok(()) + } + + #[rstest] + #[case::user(DailyTable::User, "daily-dispatch-user")] + #[case::team(DailyTable::Team, "daily-dispatch-team")] + #[case::end_user(DailyTable::EndUser, "daily-dispatch-end-user")] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_spend_dispatches_to_selected_table( + #[case] table: DailyTable, + #[case] entity_id: &'static str, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + let api_key = format!("{entity_id}-key"); + let rows = [DailySpend { + entity_id: entity_id.to_owned(), + date: "2026-02-20".to_owned(), + api_key: api_key.clone(), + model: "dispatcher-model".to_owned(), + custom_llm_provider: "dispatcher-provider".to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: "/dispatcher".to_owned(), + model_group: Some("dispatcher-group".to_owned()), + prompt_tokens: 1, + completion_tokens: 2, + api_requests: 3, + successful_requests: 4, + failed_requests: 5, + cache_read_input_tokens: 6, + cache_creation_input_tokens: 7, + total_response_time_ms: 8, + timed_requests: 9, + spend: 0.125, + }]; + upsert_daily_spend(&mut *tx, table, &rows).await?; + + let count: i64 = match table { + DailyTable::User => + { + #[expect( + clippy::disallowed_methods, + reason = "The integration test checks the user-table dispatcher route" + )] + sqlx::query_scalar( + r#"SELECT COUNT(*) FROM "LiteLLM_DailyUserSpend" + WHERE user_id = $1 AND date = $2 AND api_key = $3 AND model = $4"#, + ) + .bind(entity_id) + .bind("2026-02-20") + .bind(&api_key) + .bind("dispatcher-model") + .fetch_one(&mut *tx) + .await? + } + DailyTable::Team => + { + #[expect( + clippy::disallowed_methods, + reason = "The integration test checks the team-table dispatcher route" + )] + sqlx::query_scalar( + r#"SELECT COUNT(*) FROM "LiteLLM_DailyTeamSpend" + WHERE team_id = $1 AND date = $2 AND api_key = $3 AND model = $4"#, + ) + .bind(entity_id) + .bind("2026-02-20") + .bind(&api_key) + .bind("dispatcher-model") + .fetch_one(&mut *tx) + .await? + } + DailyTable::EndUser => + { + #[expect( + clippy::disallowed_methods, + reason = "The integration test checks the end-user-table dispatcher route" + )] + sqlx::query_scalar( + r#"SELECT COUNT(*) FROM "LiteLLM_DailyEndUserSpend" + WHERE end_user_id = $1 AND date = $2 AND api_key = $3 AND model = $4"#, + ) + .bind(entity_id) + .bind("2026-02-20") + .bind(&api_key) + .bind("dispatcher-model") + .fetch_one(&mut *tx) + .await? + } + }; + assert_eq!(count, 1); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/daily_team_spend.rs b/litellm-rust/crates/db/src/spend/daily_team_spend.rs new file mode 100644 index 00000000000..f2acd23f468 --- /dev/null +++ b/litellm-rust/crates/db/src/spend/daily_team_spend.rs @@ -0,0 +1,485 @@ +use sqlx::PgExecutor; + +use super::DailySpend; +use crate::Error; + +pub async fn upsert_daily_team_spend( + executor: impl PgExecutor<'_>, + rows: &[DailySpend], +) -> Result<(), Error> { + if rows.is_empty() { + return Ok(()); + } + macro_rules! column { + ($field:ident) => { + rows.iter() + .map(|row| row.$field.clone()) + .collect::>() + }; + } + sqlx::query_file!( + "queries/spend/upsert_daily_team_spend.sql", + &column!(entity_id), + &column!(date), + &column!(api_key), + &column!(model), + &column!(custom_llm_provider), + &column!(endpoint), + &column!(model_group) as &[Option], + &column!(prompt_tokens), + &column!(completion_tokens), + &column!(api_requests), + &column!(successful_requests), + &column!(failed_requests), + &column!(cache_read_input_tokens), + &column!(cache_creation_input_tokens), + &column!(total_response_time_ms), + &column!(timed_requests), + &column!(spend), + &column!(mcp_namespaced_tool_name), + ) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{DailySpend, upsert_daily_team_spend}; + use crate::{DailyTeamRow, test_support::database_pool}; + use rstest::rstest; + use sqlx::{PgPool, types::chrono::NaiveDateTime}; + + #[rstest] + #[case::distinct_tools("server.fetch", "server.search")] + #[case::default_and_named_tool("", "server.search")] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_team_spend_separates_tools_and_accumulates( + #[case] first_tool: &str, + #[case] second_tool: &str, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + let first = DailySpend { + entity_id: "daily-team-tools".to_owned(), + date: "2026-01-15".to_owned(), + api_key: "daily-key-tools".to_owned(), + model: "daily-model".to_owned(), + custom_llm_provider: "daily-provider".to_owned(), + endpoint: "/mcp".to_owned(), + mcp_namespaced_tool_name: first_tool.to_owned(), + api_requests: 1, + spend: 0.125, + ..Default::default() + }; + let second = DailySpend { + mcp_namespaced_tool_name: second_tool.to_owned(), + api_requests: 2, + spend: 0.25, + ..first.clone() + }; + upsert_daily_team_spend(&mut *tx, &[second.clone(), first.clone()]).await?; + upsert_daily_team_spend( + &mut *tx, + &[ + DailySpend { + api_requests: 3, + spend: 0.5, + ..first + }, + DailySpend { + api_requests: 5, + spend: 1.0, + ..second + }, + ], + ) + .await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test reads stored tool attribution and counters" + )] + let rows = sqlx::query_as::<_, DailyTeamRow>( + r#"SELECT * FROM "LiteLLM_DailyTeamSpend" + WHERE team_id = $1 AND date = $2 + ORDER BY mcp_namespaced_tool_name"#, + ) + .bind("daily-team-tools") + .bind("2026-01-15") + .fetch_all(&mut *tx) + .await?; + + assert_eq!(rows.len(), 2); + assert_eq!( + rows[0].mcp_namespaced_tool_name.as_deref(), + Some(first_tool) + ); + assert_eq!( + rows[1].mcp_namespaced_tool_name.as_deref(), + Some(second_tool) + ); + assert_eq!((rows[0].api_requests, rows[0].spend), (4, 0.625)); + assert_eq!((rows[1].api_requests, rows[1].spend), (7, 1.25)); + Ok(()) + } + + #[rstest] + #[case::full_keys( + "daily-team-base", + "daily-key-base", + "daily-model", + "daily-provider", + "/chat/completions", + Some("first-group") + )] + #[case::initial_model_group_none( + "daily-team-base", + "daily-key-base", + "daily-model", + "daily-provider", + "/chat/completions", + None + )] + #[case::empty_optional_keys("", "", "", "", "", Some(""))] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_team_spend_upsert_preserves_keys_and_accumulates( + #[case] team_id: &'static str, + #[case] api_key: &'static str, + #[case] model: &'static str, + #[case] custom_llm_provider: &'static str, + #[case] endpoint: &'static str, + #[case] first_model_group: Option<&'static str>, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test exercises UTC timestamp behavior under a non-UTC session" + )] + sqlx::query("SET LOCAL TIME ZONE 'America/Los_Angeles'") + .execute(&mut *tx) + .await?; + + let first_batch = [ + DailySpend { + entity_id: team_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: first_model_group.map(str::to_owned), + prompt_tokens: 1, + completion_tokens: 2, + api_requests: 3, + successful_requests: 4, + failed_requests: 5, + cache_read_input_tokens: 6, + cache_creation_input_tokens: 7, + total_response_time_ms: 8, + timed_requests: 9, + spend: 0.125, + }, + DailySpend { + entity_id: team_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: "endpoint-sentinel".to_owned(), + model_group: Some("sentinel-endpoint-group".to_owned()), + prompt_tokens: 101, + completion_tokens: 102, + api_requests: 103, + successful_requests: 104, + failed_requests: 105, + cache_read_input_tokens: 106, + cache_creation_input_tokens: 107, + total_response_time_ms: 108, + timed_requests: 109, + spend: 1.25, + }, + DailySpend { + entity_id: team_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: "api-key-sentinel".to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("sentinel-key-group".to_owned()), + prompt_tokens: 201, + completion_tokens: 202, + api_requests: 203, + successful_requests: 204, + failed_requests: 205, + cache_read_input_tokens: 206, + cache_creation_input_tokens: 207, + total_response_time_ms: 208, + timed_requests: 209, + spend: 2.5, + }, + DailySpend { + entity_id: "entity-sentinel".to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("sentinel-entity-group".to_owned()), + prompt_tokens: 301, + completion_tokens: 302, + api_requests: 303, + successful_requests: 304, + failed_requests: 305, + cache_read_input_tokens: 306, + cache_creation_input_tokens: 307, + total_response_time_ms: 308, + timed_requests: 309, + spend: 3.75, + }, + ]; + upsert_daily_team_spend(&mut *tx, &first_batch).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test captures the inserted row identity before conflict handling" + )] + let (initial_id, initial_created_at): (String, NaiveDateTime) = sqlx::query_as( + r#"SELECT id, created_at + FROM "LiteLLM_DailyTeamSpend" + WHERE team_id = $1 AND date = $2 AND api_key = $3 AND model = $4 + AND custom_llm_provider = $5 AND endpoint = $6"#, + ) + .bind(team_id) + .bind("2026-01-15") + .bind(api_key) + .bind(model) + .bind(custom_llm_provider) + .bind(endpoint) + .fetch_one(&mut *tx) + .await?; + assert!(!initial_id.is_empty()); + + #[expect( + clippy::disallowed_methods, + reason = "The integration test seeds updated_at before exercising the conflict update" + )] + let seeded_updated_at: NaiveDateTime = sqlx::query_scalar( + r#"UPDATE "LiteLLM_DailyTeamSpend" + SET updated_at = TIMESTAMP '2000-01-01' + WHERE id = $1 + RETURNING updated_at"#, + ) + .bind(&initial_id) + .fetch_one(&mut *tx) + .await?; + + let increment = [DailySpend { + entity_id: team_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("second-group".to_owned()), + prompt_tokens: 10, + completion_tokens: 20, + api_requests: 30, + successful_requests: 40, + failed_requests: 50, + cache_read_input_tokens: 60, + cache_creation_input_tokens: 70, + total_response_time_ms: 80, + timed_requests: 90, + spend: 0.625, + }]; + upsert_daily_team_spend(&mut *tx, &increment).await?; + upsert_daily_team_spend(&mut *tx, &[]).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test captures the transaction timestamp for assertions" + )] + let expected_updated_at: NaiveDateTime = + sqlx::query_scalar("SELECT (NOW() AT TIME ZONE 'UTC')::timestamp(3)") + .fetch_one(&mut *tx) + .await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test reads all rows to verify attribution and accumulation" + )] + let rows: Vec = sqlx::query_as( + r#"SELECT id, team_id, api_key, model, custom_llm_provider, endpoint, + model_group, mcp_namespaced_tool_name, prompt_tokens, + completion_tokens, api_requests, successful_requests, + failed_requests, cache_read_input_tokens, + cache_creation_input_tokens, total_response_time_ms, + timed_requests, spend, date, created_at, updated_at + FROM "LiteLLM_DailyTeamSpend" + WHERE date = $1 AND model = $2 AND custom_llm_provider = $3 + AND mcp_namespaced_tool_name = '' AND team_id = ANY($4) + ORDER BY CASE + WHEN team_id = $5 AND api_key = $6 AND endpoint = $7 THEN 0 + WHEN team_id = $5 AND api_key = $6 AND endpoint = $9 THEN 1 + WHEN team_id = $5 AND api_key = $8 AND endpoint = $7 THEN 2 + WHEN team_id = $10 AND api_key = $6 AND endpoint = $7 THEN 3 + ELSE 4 + END"#, + ) + .bind("2026-01-15") + .bind(model) + .bind(custom_llm_provider) + .bind(vec![team_id, "entity-sentinel"]) + .bind(team_id) + .bind(api_key) + .bind(endpoint) + .bind("api-key-sentinel") + .bind("endpoint-sentinel") + .bind("entity-sentinel") + .fetch_all(&mut *tx) + .await?; + + assert_eq!(rows.len(), 4); + assert_eq!(rows[0].id, initial_id); + assert_eq!(rows[0].team_id.as_deref(), Some(team_id)); + assert_eq!(rows[0].api_key, api_key); + assert_eq!(rows[0].model.as_deref(), Some(model)); + assert_eq!( + rows[0].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[0].endpoint.as_deref(), Some(endpoint)); + assert_eq!(rows[0].model_group.as_deref(), first_model_group); + assert_eq!(rows[0].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[0].prompt_tokens, + rows[0].completion_tokens, + rows[0].api_requests, + rows[0].successful_requests, + rows[0].failed_requests, + rows[0].cache_read_input_tokens, + rows[0].cache_creation_input_tokens, + rows[0].total_response_time_ms, + rows[0].timed_requests, + rows[0].spend + ), + (11, 22, 33, 44, 55, 66, 77, 88, 99, 0.75) + ); + assert_eq!(rows[0].date, "2026-01-15"); + assert_eq!(rows[0].created_at, initial_created_at); + assert!(rows[0].updated_at > seeded_updated_at); + assert_eq!(rows[0].updated_at, expected_updated_at); + + assert!(!rows[1].id.is_empty()); + assert_eq!(rows[1].team_id.as_deref(), Some(team_id)); + assert_eq!(rows[1].api_key, api_key); + assert_eq!(rows[1].model.as_deref(), Some(model)); + assert_eq!( + rows[1].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[1].endpoint.as_deref(), Some("endpoint-sentinel")); + assert_eq!( + rows[1].model_group.as_deref(), + Some("sentinel-endpoint-group") + ); + assert_eq!(rows[1].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[1].prompt_tokens, + rows[1].completion_tokens, + rows[1].api_requests, + rows[1].successful_requests, + rows[1].failed_requests, + rows[1].cache_read_input_tokens, + rows[1].cache_creation_input_tokens, + rows[1].total_response_time_ms, + rows[1].timed_requests, + rows[1].spend + ), + (101, 102, 103, 104, 105, 106, 107, 108, 109, 1.25) + ); + assert_eq!(rows[1].date, "2026-01-15"); + assert!(rows[1].updated_at > seeded_updated_at); + assert_eq!(rows[1].updated_at, expected_updated_at); + + assert!(!rows[2].id.is_empty()); + assert_eq!(rows[2].team_id.as_deref(), Some(team_id)); + assert_eq!(rows[2].api_key, "api-key-sentinel"); + assert_eq!(rows[2].model.as_deref(), Some(model)); + assert_eq!( + rows[2].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[2].endpoint.as_deref(), Some(endpoint)); + assert_eq!(rows[2].model_group.as_deref(), Some("sentinel-key-group")); + assert_eq!(rows[2].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[2].prompt_tokens, + rows[2].completion_tokens, + rows[2].api_requests, + rows[2].successful_requests, + rows[2].failed_requests, + rows[2].cache_read_input_tokens, + rows[2].cache_creation_input_tokens, + rows[2].total_response_time_ms, + rows[2].timed_requests, + rows[2].spend + ), + (201, 202, 203, 204, 205, 206, 207, 208, 209, 2.5) + ); + assert_eq!(rows[2].date, "2026-01-15"); + assert!(rows[2].updated_at > seeded_updated_at); + assert_eq!(rows[2].updated_at, expected_updated_at); + + assert!(!rows[3].id.is_empty()); + assert_eq!(rows[3].team_id.as_deref(), Some("entity-sentinel")); + assert_eq!(rows[3].api_key, api_key); + assert_eq!(rows[3].model.as_deref(), Some(model)); + assert_eq!( + rows[3].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[3].endpoint.as_deref(), Some(endpoint)); + assert_eq!( + rows[3].model_group.as_deref(), + Some("sentinel-entity-group") + ); + assert_eq!(rows[3].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[3].prompt_tokens, + rows[3].completion_tokens, + rows[3].api_requests, + rows[3].successful_requests, + rows[3].failed_requests, + rows[3].cache_read_input_tokens, + rows[3].cache_creation_input_tokens, + rows[3].total_response_time_ms, + rows[3].timed_requests, + rows[3].spend + ), + (301, 302, 303, 304, 305, 306, 307, 308, 309, 3.75) + ); + assert_eq!(rows[3].date, "2026-01-15"); + assert!(rows[3].updated_at > seeded_updated_at); + assert_eq!(rows[3].updated_at, expected_updated_at); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/daily_user_spend.rs b/litellm-rust/crates/db/src/spend/daily_user_spend.rs new file mode 100644 index 00000000000..3530eb245c5 --- /dev/null +++ b/litellm-rust/crates/db/src/spend/daily_user_spend.rs @@ -0,0 +1,500 @@ +use sqlx::PgExecutor; + +use crate::Error; + +#[macro_rules_attribute::apply(crate::db_input)] +#[derive(Default)] +pub struct DailySpend { + pub entity_id: String, + pub date: String, + pub api_key: String, + pub model: String, + pub custom_llm_provider: String, + pub mcp_namespaced_tool_name: String, + pub endpoint: String, + pub model_group: Option, + pub prompt_tokens: i64, + pub completion_tokens: i64, + pub api_requests: i64, + pub successful_requests: i64, + pub failed_requests: i64, + pub cache_read_input_tokens: i64, + pub cache_creation_input_tokens: i64, + pub total_response_time_ms: i64, + pub timed_requests: i64, + pub spend: f64, +} + +pub async fn upsert_daily_user_spend( + executor: impl PgExecutor<'_>, + rows: &[DailySpend], +) -> Result<(), Error> { + if rows.is_empty() { + return Ok(()); + } + macro_rules! column { + ($field:ident) => { + rows.iter() + .map(|row| row.$field.clone()) + .collect::>() + }; + } + sqlx::query_file!( + "queries/spend/upsert_daily_user_spend.sql", + &column!(entity_id), + &column!(date), + &column!(api_key), + &column!(model), + &column!(custom_llm_provider), + &column!(endpoint), + &column!(model_group) as &[Option], + &column!(prompt_tokens), + &column!(completion_tokens), + &column!(api_requests), + &column!(successful_requests), + &column!(failed_requests), + &column!(cache_read_input_tokens), + &column!(cache_creation_input_tokens), + &column!(total_response_time_ms), + &column!(timed_requests), + &column!(spend), + &column!(mcp_namespaced_tool_name), + ) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{DailySpend, upsert_daily_user_spend}; + use crate::{DailyUserRow, test_support::database_pool}; + use rstest::rstest; + use sqlx::{PgPool, types::chrono::NaiveDateTime}; + + #[rstest] + #[case::distinct_tools("server/search", "server/write")] + #[case::default_empty_tool("", "server/search")] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_user_spend_separates_tools_and_accumulates( + #[case] first_tool: &str, + #[case] second_tool: &str, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + let first = DailySpend { + entity_id: format!("daily-user-tools-{first_tool}"), + date: "2026-01-15".to_owned(), + api_key: "tool-key".to_owned(), + model: "tool-model".to_owned(), + custom_llm_provider: "tool-provider".to_owned(), + endpoint: "/mcp".to_owned(), + model_group: Some("tool-group".to_owned()), + mcp_namespaced_tool_name: first_tool.to_owned(), + prompt_tokens: 3, + api_requests: 1, + spend: 0.25, + ..Default::default() + }; + let second = DailySpend { + mcp_namespaced_tool_name: second_tool.to_owned(), + prompt_tokens: 7, + api_requests: 2, + spend: 0.5, + ..first.clone() + }; + upsert_daily_user_spend(&mut *tx, &[second.clone(), first.clone()]).await?; + upsert_daily_user_spend(&mut *tx, &[first.clone(), second]).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test reads stored rows to verify tool attribution" + )] + let rows: Vec = sqlx::query_as( + r#"SELECT * FROM "LiteLLM_DailyUserSpend" + WHERE user_id = $1 + ORDER BY mcp_namespaced_tool_name, endpoint"#, + ) + .bind(&first.entity_id) + .fetch_all(&mut *tx) + .await?; + + assert_eq!(rows.len(), 2); + assert_eq!( + rows[0].mcp_namespaced_tool_name.as_deref(), + Some(first_tool) + ); + assert_eq!( + rows[1].mcp_namespaced_tool_name.as_deref(), + Some(second_tool) + ); + assert_eq!( + (rows[0].prompt_tokens, rows[0].api_requests, rows[0].spend), + (6, 2, 0.5) + ); + assert_eq!( + (rows[1].prompt_tokens, rows[1].api_requests, rows[1].spend), + (14, 4, 1.0) + ); + Ok(()) + } + + #[rstest] + #[case::full_keys( + "daily-user-base", + "daily-key-base", + "daily-model", + "daily-provider", + "/chat/completions", + Some("first-group") + )] + #[case::initial_model_group_none( + "daily-user-base", + "daily-key-base", + "daily-model", + "daily-provider", + "/chat/completions", + None + )] + #[case::empty_optional_keys("", "", "", "", "", Some(""))] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn daily_user_spend_upsert_preserves_keys_and_accumulates( + #[case] user_id: &'static str, + #[case] api_key: &'static str, + #[case] model: &'static str, + #[case] custom_llm_provider: &'static str, + #[case] endpoint: &'static str, + #[case] first_model_group: Option<&'static str>, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test exercises UTC timestamp behavior under a non-UTC session" + )] + sqlx::query("SET LOCAL TIME ZONE 'America/Los_Angeles'") + .execute(&mut *tx) + .await?; + + let first_batch = [ + DailySpend { + entity_id: user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: first_model_group.map(str::to_owned), + prompt_tokens: 1, + completion_tokens: 2, + api_requests: 3, + successful_requests: 4, + failed_requests: 5, + cache_read_input_tokens: 6, + cache_creation_input_tokens: 7, + total_response_time_ms: 8, + timed_requests: 9, + spend: 0.125, + }, + DailySpend { + entity_id: user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: "endpoint-sentinel".to_owned(), + model_group: Some("sentinel-endpoint-group".to_owned()), + prompt_tokens: 101, + completion_tokens: 102, + api_requests: 103, + successful_requests: 104, + failed_requests: 105, + cache_read_input_tokens: 106, + cache_creation_input_tokens: 107, + total_response_time_ms: 108, + timed_requests: 109, + spend: 1.25, + }, + DailySpend { + entity_id: user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: "api-key-sentinel".to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("sentinel-key-group".to_owned()), + prompt_tokens: 201, + completion_tokens: 202, + api_requests: 203, + successful_requests: 204, + failed_requests: 205, + cache_read_input_tokens: 206, + cache_creation_input_tokens: 207, + total_response_time_ms: 208, + timed_requests: 209, + spend: 2.5, + }, + DailySpend { + entity_id: "entity-sentinel".to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("sentinel-entity-group".to_owned()), + prompt_tokens: 301, + completion_tokens: 302, + api_requests: 303, + successful_requests: 304, + failed_requests: 305, + cache_read_input_tokens: 306, + cache_creation_input_tokens: 307, + total_response_time_ms: 308, + timed_requests: 309, + spend: 3.75, + }, + ]; + upsert_daily_user_spend(&mut *tx, &first_batch).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test captures the inserted row identity before conflict handling" + )] + let (initial_id, initial_created_at): (String, NaiveDateTime) = sqlx::query_as( + r#"SELECT id, created_at + FROM "LiteLLM_DailyUserSpend" + WHERE user_id = $1 AND date = $2 AND api_key = $3 AND model = $4 + AND custom_llm_provider = $5 AND endpoint = $6"#, + ) + .bind(user_id) + .bind("2026-01-15") + .bind(api_key) + .bind(model) + .bind(custom_llm_provider) + .bind(endpoint) + .fetch_one(&mut *tx) + .await?; + assert!(!initial_id.is_empty()); + + #[expect( + clippy::disallowed_methods, + reason = "The integration test seeds updated_at before exercising the conflict update" + )] + let seeded_updated_at: NaiveDateTime = sqlx::query_scalar( + r#"UPDATE "LiteLLM_DailyUserSpend" + SET updated_at = TIMESTAMP '2000-01-01' + WHERE id = $1 + RETURNING updated_at"#, + ) + .bind(&initial_id) + .fetch_one(&mut *tx) + .await?; + + let increment = [DailySpend { + entity_id: user_id.to_owned(), + date: "2026-01-15".to_owned(), + api_key: api_key.to_owned(), + model: model.to_owned(), + custom_llm_provider: custom_llm_provider.to_owned(), + mcp_namespaced_tool_name: String::new(), + endpoint: endpoint.to_owned(), + model_group: Some("second-group".to_owned()), + prompt_tokens: 10, + completion_tokens: 20, + api_requests: 30, + successful_requests: 40, + failed_requests: 50, + cache_read_input_tokens: 60, + cache_creation_input_tokens: 70, + total_response_time_ms: 80, + timed_requests: 90, + spend: 0.625, + }]; + upsert_daily_user_spend(&mut *tx, &increment).await?; + upsert_daily_user_spend(&mut *tx, &[]).await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test captures the transaction timestamp for assertions" + )] + let expected_updated_at: NaiveDateTime = + sqlx::query_scalar("SELECT (NOW() AT TIME ZONE 'UTC')::timestamp(3)") + .fetch_one(&mut *tx) + .await?; + + #[expect( + clippy::disallowed_methods, + reason = "The integration test reads all rows to verify attribution and accumulation" + )] + let rows: Vec = sqlx::query_as( + r#"SELECT id, user_id, api_key, model, custom_llm_provider, endpoint, + model_group, mcp_namespaced_tool_name, prompt_tokens, + completion_tokens, api_requests, successful_requests, + failed_requests, cache_read_input_tokens, + cache_creation_input_tokens, total_response_time_ms, + timed_requests, spend, date, created_at, updated_at + FROM "LiteLLM_DailyUserSpend" + WHERE date = $1 AND model = $2 AND custom_llm_provider = $3 + AND mcp_namespaced_tool_name = '' AND user_id = ANY($4) + ORDER BY CASE + WHEN user_id = $5 AND api_key = $6 AND endpoint = $7 THEN 0 + WHEN user_id = $5 AND api_key = $6 AND endpoint = $9 THEN 1 + WHEN user_id = $5 AND api_key = $8 AND endpoint = $7 THEN 2 + WHEN user_id = $10 AND api_key = $6 AND endpoint = $7 THEN 3 + ELSE 4 + END"#, + ) + .bind("2026-01-15") + .bind(model) + .bind(custom_llm_provider) + .bind(vec![user_id, "entity-sentinel"]) + .bind(user_id) + .bind(api_key) + .bind(endpoint) + .bind("api-key-sentinel") + .bind("endpoint-sentinel") + .bind("entity-sentinel") + .fetch_all(&mut *tx) + .await?; + + assert_eq!(rows.len(), 4); + assert_eq!(rows[0].id, initial_id); + assert_eq!(rows[0].user_id.as_deref(), Some(user_id)); + assert_eq!(rows[0].api_key, api_key); + assert_eq!(rows[0].model.as_deref(), Some(model)); + assert_eq!( + rows[0].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[0].endpoint.as_deref(), Some(endpoint)); + assert_eq!(rows[0].model_group.as_deref(), first_model_group); + assert_eq!(rows[0].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[0].prompt_tokens, + rows[0].completion_tokens, + rows[0].api_requests, + rows[0].successful_requests, + rows[0].failed_requests, + rows[0].cache_read_input_tokens, + rows[0].cache_creation_input_tokens, + rows[0].total_response_time_ms, + rows[0].timed_requests, + rows[0].spend + ), + (11, 22, 33, 44, 55, 66, 77, 88, 99, 0.75) + ); + assert_eq!(rows[0].date, "2026-01-15"); + assert_eq!(rows[0].created_at, initial_created_at); + assert!(rows[0].updated_at > seeded_updated_at); + assert_eq!(rows[0].updated_at, expected_updated_at); + + assert!(!rows[1].id.is_empty()); + assert_eq!(rows[1].user_id.as_deref(), Some(user_id)); + assert_eq!(rows[1].api_key, api_key); + assert_eq!(rows[1].model.as_deref(), Some(model)); + assert_eq!( + rows[1].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[1].endpoint.as_deref(), Some("endpoint-sentinel")); + assert_eq!( + rows[1].model_group.as_deref(), + Some("sentinel-endpoint-group") + ); + assert_eq!(rows[1].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[1].prompt_tokens, + rows[1].completion_tokens, + rows[1].api_requests, + rows[1].successful_requests, + rows[1].failed_requests, + rows[1].cache_read_input_tokens, + rows[1].cache_creation_input_tokens, + rows[1].total_response_time_ms, + rows[1].timed_requests, + rows[1].spend + ), + (101, 102, 103, 104, 105, 106, 107, 108, 109, 1.25) + ); + assert_eq!(rows[1].date, "2026-01-15"); + assert!(rows[1].updated_at > seeded_updated_at); + assert_eq!(rows[1].updated_at, expected_updated_at); + + assert!(!rows[2].id.is_empty()); + assert_eq!(rows[2].user_id.as_deref(), Some(user_id)); + assert_eq!(rows[2].api_key, "api-key-sentinel"); + assert_eq!(rows[2].model.as_deref(), Some(model)); + assert_eq!( + rows[2].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[2].endpoint.as_deref(), Some(endpoint)); + assert_eq!(rows[2].model_group.as_deref(), Some("sentinel-key-group")); + assert_eq!(rows[2].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[2].prompt_tokens, + rows[2].completion_tokens, + rows[2].api_requests, + rows[2].successful_requests, + rows[2].failed_requests, + rows[2].cache_read_input_tokens, + rows[2].cache_creation_input_tokens, + rows[2].total_response_time_ms, + rows[2].timed_requests, + rows[2].spend + ), + (201, 202, 203, 204, 205, 206, 207, 208, 209, 2.5) + ); + assert_eq!(rows[2].date, "2026-01-15"); + assert!(rows[2].updated_at > seeded_updated_at); + assert_eq!(rows[2].updated_at, expected_updated_at); + + assert!(!rows[3].id.is_empty()); + assert_eq!(rows[3].user_id.as_deref(), Some("entity-sentinel")); + assert_eq!(rows[3].api_key, api_key); + assert_eq!(rows[3].model.as_deref(), Some(model)); + assert_eq!( + rows[3].custom_llm_provider.as_deref(), + Some(custom_llm_provider) + ); + assert_eq!(rows[3].endpoint.as_deref(), Some(endpoint)); + assert_eq!( + rows[3].model_group.as_deref(), + Some("sentinel-entity-group") + ); + assert_eq!(rows[3].mcp_namespaced_tool_name.as_deref(), Some("")); + assert_eq!( + ( + rows[3].prompt_tokens, + rows[3].completion_tokens, + rows[3].api_requests, + rows[3].successful_requests, + rows[3].failed_requests, + rows[3].cache_read_input_tokens, + rows[3].cache_creation_input_tokens, + rows[3].total_response_time_ms, + rows[3].timed_requests, + rows[3].spend + ), + (301, 302, 303, 304, 305, 306, 307, 308, 309, 3.75) + ); + assert_eq!(rows[3].date, "2026-01-15"); + assert!(rows[3].updated_at > seeded_updated_at); + assert_eq!(rows[3].updated_at, expected_updated_at); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/end_user_spend.rs b/litellm-rust/crates/db/src/spend/end_user_spend.rs new file mode 100644 index 00000000000..ca327ac7e18 --- /dev/null +++ b/litellm-rust/crates/db/src/spend/end_user_spend.rs @@ -0,0 +1,197 @@ +use sqlx::PgExecutor; + +use super::EntitySpend; +use crate::Error; + +pub async fn add_end_user_spend( + executor: impl PgExecutor<'_>, + spend: &EntitySpend, +) -> Result<(), Error> { + spend.validate()?; + if spend.is_empty() { + return Ok(()); + } + sqlx::query_file!( + "queries/spend/add_end_user_spend.sql", + &spend.ids, + &spend.costs + ) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{EntitySpend, add_end_user_spend}; + use crate::test_support::database_pool; + use rstest::rstest; + use sqlx::PgPool; + + #[rstest] + #[case::fractional_costs(0.25, 0.75, false, 2.5, 1.5)] + #[case::zero_costs(0.0, 0.0, false, 2.0, 0.0)] + #[case::duplicate_ids(0.25, 0.75, true, 3.0, 3.0)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn end_user_spend_matches_python_upsert( + #[case] existing_cost: f64, + #[case] new_cost: f64, + #[case] duplicate_ids: bool, + #[case] expected_existing_spend: f64, + #[case] expected_new_spend: f64, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _budget = sqlx::query( + r#"INSERT INTO "LiteLLM_BudgetTable" (budget_id, created_by, updated_by) + VALUES ('end-user-spend-budget', 'db-query-test', 'db-query-test')"#, + ) + .execute(&mut *tx) + .await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _users = sqlx::query( + r#"INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked, budget_id) + VALUES ('end-user-spend-z', 2, true, 'end-user-spend-budget'), + ('end-user-spend-untouched', 100, true, NULL)"#, + ) + .execute(&mut *tx) + .await?; + let spend = if duplicate_ids { + EntitySpend { + ids: vec![ + "end-user-spend-z".into(), + "end-user-spend-a".into(), + "end-user-spend-z".into(), + "end-user-spend-a".into(), + ], + costs: vec![existing_cost, new_cost, existing_cost, new_cost], + } + } else { + EntitySpend { + ids: vec!["end-user-spend-z".into(), "end-user-spend-a".into()], + costs: vec![existing_cost, new_cost], + } + }; + add_end_user_spend(&mut *tx, &spend).await?; + add_end_user_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, f64, bool, Option)> = sqlx::query_as( + r#"SELECT user_id, spend, blocked, budget_id FROM "LiteLLM_EndUserTable" + WHERE user_id = ANY($1) ORDER BY user_id"#, + ) + .bind(vec![ + "end-user-spend-a", + "end-user-spend-z", + "end-user-spend-untouched", + ]) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows, + vec![ + ("end-user-spend-a".into(), expected_new_spend, false, None), + ("end-user-spend-untouched".into(), 100.0, true, None), + ( + "end-user-spend-z".into(), + expected_existing_spend, + true, + Some("end-user-spend-budget".into()), + ), + ] + ); + Ok(()) + } + + #[rstest] + #[case::missing_cost(vec!["end-user-invalid-existing".into(), "end-user-invalid-new".into()], vec![1.0])] + #[case::extra_cost(vec!["end-user-invalid-existing".into()], vec![1.0, 2.0])] + #[case::empty_ids(vec![], vec![1.0])] + #[case::empty_costs(vec!["end-user-invalid-existing".into()], vec![])] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn malformed_end_user_spend_is_rejected_before_writes( + #[case] ids: Vec, + #[case] costs: Vec, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _user = sqlx::query( + r#"INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked) + VALUES ('end-user-invalid-existing', 8.5, true)"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { ids, costs }; + let result = add_end_user_spend(&mut *tx, &spend).await; + assert!(matches!( + result, + Err(crate::Error::InvalidSpendBatch { ids, costs }) + if ids == spend.ids.len() && costs == spend.costs.len() + )); + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, f64, bool)> = sqlx::query_as( + r#"SELECT user_id, spend, blocked FROM "LiteLLM_EndUserTable" + WHERE user_id IN ('end-user-invalid-existing', 'end-user-invalid-new')"#, + ) + .fetch_all(&mut *tx) + .await?; + assert_eq!(rows, vec![("end-user-invalid-existing".into(), 8.5, true)]); + Ok(()) + } + + #[rstest] + #[case::empty_batch()] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn empty_end_user_spend_is_a_noop( + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _user = sqlx::query( + r#"INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked) + VALUES ('end-user-empty-spend', 8.5, true)"#, + ) + .execute(&mut *tx) + .await?; + add_end_user_spend(&mut *tx, &EntitySpend::default()).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let row: (f64, bool, Option) = sqlx::query_as( + r#"SELECT spend, blocked, budget_id FROM "LiteLLM_EndUserTable" + WHERE user_id = 'end-user-empty-spend'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(row, (8.5, true, None)); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/key_spend.rs b/litellm-rust/crates/db/src/spend/key_spend.rs new file mode 100644 index 00000000000..5a6ba13307c --- /dev/null +++ b/litellm-rust/crates/db/src/spend/key_spend.rs @@ -0,0 +1,309 @@ +use sqlx::PgExecutor; +use std::collections::BTreeMap; + +use crate::Error; + +#[macro_rules_attribute::apply(crate::db_input)] +#[derive(Default)] +pub struct EntitySpend { + pub ids: Vec, + pub costs: Vec, +} + +impl EntitySpend { + pub(crate) fn validate(&self) -> Result<(), Error> { + if self.ids.len() != self.costs.len() { + return Err(Error::InvalidSpendBatch { + ids: self.ids.len(), + costs: self.costs.len(), + }); + } + Ok(()) + } + + pub fn is_empty(&self) -> bool { + self.ids.is_empty() + } +} + +impl FromIterator<(String, f64)> for EntitySpend { + fn from_iter>(entries: I) -> Self { + let entries = entries + .into_iter() + .fold(BTreeMap::new(), |mut sums, (id, cost)| { + *sums.entry(id).or_insert(0.0) += cost; + sums + }); + let (ids, costs) = entries.into_iter().unzip(); + Self { ids, costs } + } +} + +pub async fn add_key_spend( + executor: impl PgExecutor<'_>, + spend: &EntitySpend, +) -> Result<(), Error> { + spend.validate()?; + if spend.is_empty() { + return Ok(()); + } + sqlx::query_file!("queries/spend/add_key_spend.sql", &spend.ids, &spend.costs) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{EntitySpend, add_key_spend}; + use crate::{Error, test_support::database_pool}; + use rstest::rstest; + use sqlx::PgPool; + + #[rstest] + #[case::direct(false)] + #[case::collected(true)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn duplicate_key_spend_accumulates( + #[case] collected: bool, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + sqlx::query( + r#"INSERT INTO "LiteLLM_VerificationToken" (token, spend, total_spend, updated_at) + VALUES ('key-spend-duplicate', 2, 7, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = if collected { + [ + ("key-spend-duplicate".into(), 0.25), + ("key-spend-duplicate".into(), 0.75), + ] + .into_iter() + .collect() + } else { + EntitySpend { + ids: vec!["key-spend-duplicate".into(), "key-spend-duplicate".into()], + costs: vec![0.25, 0.75], + } + }; + add_key_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let row: (f64, f64) = sqlx::query_as( + r#"SELECT spend, total_spend FROM "LiteLLM_VerificationToken" + WHERE token = 'key-spend-duplicate'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(row, (3.0, 8.0)); + Ok(()) + } + + #[rstest] + #[case::extra_id(vec!["key-spend-invalid".into(), "key-spend-invalid".into()], vec![0.25])] + #[case::extra_cost(vec!["key-spend-invalid".into()], vec![0.25, 0.75])] + #[case::empty_ids(vec![], vec![0.25])] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn malformed_key_spend_leaves_transaction_usable( + #[case] ids: Vec, + #[case] costs: Vec, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + sqlx::query( + r#"INSERT INTO "LiteLLM_VerificationToken" + (token, spend, total_spend, last_active, updated_at) + VALUES ('key-spend-invalid', 2, 7, NULL, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { ids, costs }; + let result = add_key_spend(&mut *tx, &spend).await; + assert!(matches!( + result, + Err(Error::InvalidSpendBatch { ids, costs }) + if ids == spend.ids.len() && costs == spend.costs.len() + )); + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let unchanged: (f64, f64, bool, bool) = sqlx::query_as( + r#"SELECT spend, total_spend, last_active IS NULL, + updated_at = TIMESTAMP '2000-01-01' + FROM "LiteLLM_VerificationToken" WHERE token = 'key-spend-invalid'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(unchanged, (2.0, 7.0, true, true)); + let valid = EntitySpend { + ids: vec!["key-spend-invalid".into()], + costs: vec![0.5], + }; + add_key_spend(&mut *tx, &valid).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let updated: (f64, f64) = sqlx::query_as( + r#"SELECT spend, total_spend FROM "LiteLLM_VerificationToken" + WHERE token = 'key-spend-invalid'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(updated, (2.5, 7.5)); + Ok(()) + } + + #[rstest] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn key_activity_uses_update_time_in_old_transaction( + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + sqlx::query( + r#"INSERT INTO "LiteLLM_VerificationToken" (token, spend, total_spend, updated_at) + VALUES ('key-spend-old-transaction', 2, 7, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + #[expect( + clippy::disallowed_methods, + reason = "Transaction timing setup is outside the compile-checked query set" + )] + sqlx::query("SELECT pg_sleep(0.02)") + .execute(&mut *tx) + .await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let boundary: chrono::NaiveDateTime = sqlx::query_scalar( + "SELECT date_trunc('milliseconds', clock_timestamp() AT TIME ZONE 'UTC')", + ) + .fetch_one(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec!["key-spend-old-transaction".into()], + costs: vec![0.25], + }; + add_key_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let row: (chrono::NaiveDateTime, chrono::NaiveDateTime) = sqlx::query_as( + r#"SELECT last_active, updated_at FROM "LiteLLM_VerificationToken" + WHERE token = 'key-spend-old-transaction'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert!(row.0 >= boundary, "last_active predates update boundary"); + assert!(row.1 >= boundary, "updated_at predates update boundary"); + Ok(()) + } + + #[rstest] + #[case::fractional_costs(0.25, 2.5, 7.5, 4.0, 12.0)] + #[case::zero_costs(0.0, 2.0, 7.0, 3.0, 11.0)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn key_spend_matches_python_update_many( + #[case] cost: f64, + #[case] expected_a_spend: f64, + #[case] expected_a_total: f64, + #[case] expected_z_spend: f64, + #[case] expected_z_total: f64, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_VerificationToken" + (token, spend, total_spend, last_active, updated_at) + VALUES ('key-spend-a', 2, 7, NULL, '2000-01-01'), + ('key-spend-z', 3, 11, NULL, '2000-01-01'), + ('key-spend-untouched', 100, 200, NULL, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec![ + "key-spend-z".into(), + "key-spend-a".into(), + "key-spend-missing".into(), + ], + costs: vec![2.0 * cost, cost, 50.0], + }; + add_key_spend(&mut *tx, &EntitySpend::default()).await?; + add_key_spend(&mut *tx, &spend).await?; + add_key_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, f64, f64, bool, bool)> = sqlx::query_as( + r#"SELECT token, spend, total_spend, last_active IS NOT NULL, + updated_at > TIMESTAMP '2000-01-01' + FROM "LiteLLM_VerificationToken" + WHERE token = ANY($1) ORDER BY token"#, + ) + .bind(vec![ + "key-spend-a", + "key-spend-z", + "key-spend-missing", + "key-spend-untouched", + ]) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows, + vec![ + ( + "key-spend-a".into(), + expected_a_spend, + expected_a_total, + true, + true + ), + ("key-spend-untouched".into(), 100.0, 200.0, false, false), + ( + "key-spend-z".into(), + expected_z_spend, + expected_z_total, + true, + true + ), + ] + ); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/mod.rs b/litellm-rust/crates/db/src/spend/mod.rs new file mode 100644 index 00000000000..da255843547 --- /dev/null +++ b/litellm-rust/crates/db/src/spend/mod.rs @@ -0,0 +1,21 @@ +mod daily_end_user_spend; +mod daily_team_spend; +mod daily_user_spend; +mod end_user_spend; +mod key_spend; +mod spend_logs; +mod team_lock; +mod team_member_spend; +mod team_spend; +mod user_spend; + +pub use daily_end_user_spend::{DailyTable, upsert_daily_end_user_spend, upsert_daily_spend}; +pub use daily_team_spend::upsert_daily_team_spend; +pub use daily_user_spend::{DailySpend, upsert_daily_user_spend}; +pub use end_user_spend::add_end_user_spend; +pub use key_spend::{EntitySpend, add_key_spend}; +pub use spend_logs::{SpendLog, SpendLogRow, insert_spend_logs}; +pub use team_lock::{begin, lock_team}; +pub use team_member_spend::{TeamMemberSpend, add_team_member_spend}; +pub use team_spend::add_team_spend; +pub use user_spend::add_user_spend; diff --git a/litellm-rust/crates/db/src/spend/spend_logs.rs b/litellm-rust/crates/db/src/spend/spend_logs.rs new file mode 100644 index 00000000000..57a9ff85301 --- /dev/null +++ b/litellm-rust/crates/db/src/spend/spend_logs.rs @@ -0,0 +1,517 @@ +use chrono::{DateTime, NaiveDateTime}; +use sqlx::PgExecutor; + +use crate::Error; + +#[macro_rules_attribute::apply(crate::db_input)] +#[derive(Default)] +pub struct SpendLog { + pub request_id: String, + pub call_type: String, + pub api_key: String, + pub spend: f64, + pub total_tokens: i32, + pub prompt_tokens: i32, + pub completion_tokens: i32, + pub start_time: String, + pub end_time: String, + pub completion_start_time: Option, + pub request_duration_ms: Option, + pub model: String, + pub model_id: String, + pub model_group: String, + pub custom_llm_provider: String, + pub api_base: String, + pub user: String, + pub metadata: serde_json::Value, + pub cache_hit: String, + pub cache_key: String, + pub team_id: String, + pub organization_id: String, + pub end_user: String, + pub requester_ip_address: Option, + pub session_id: Option, + pub status: String, + pub litellm_call_id: Option, +} + +#[macro_rules_attribute::apply(crate::db_row)] +pub struct SpendLogRow { + pub request_id: String, + pub call_type: String, + pub api_key: String, + pub spend: f64, + pub total_tokens: i32, + pub prompt_tokens: i32, + pub completion_tokens: i32, + #[sqlx(rename = "startTime")] + pub start_time: NaiveDateTime, + #[sqlx(rename = "endTime")] + pub end_time: NaiveDateTime, + #[sqlx(rename = "completionStartTime")] + pub completion_start_time: Option, + pub request_duration_ms: Option, + pub model: String, + pub model_id: Option, + pub model_group: Option, + pub custom_llm_provider: Option, + pub api_base: Option, + pub user: Option, + pub metadata: Option, + pub cache_hit: Option, + pub cache_key: Option, + pub request_tags: Option, + pub team_id: Option, + pub organization_id: Option, + pub end_user: Option, + pub requester_ip_address: Option, + pub messages: Option, + pub response: Option, + pub proxy_server_request: Option, + pub session_id: Option, + pub status: Option, + pub litellm_call_id: Option, +} + +fn parse_timestamp(value: &str) -> Result { + DateTime::parse_from_rfc3339(value) + .map(|timestamp| timestamp.naive_utc()) + .or_else(|_| NaiveDateTime::parse_from_str(value, "%Y-%m-%dT%H:%M:%S%.f")) + .or_else(|_| NaiveDateTime::parse_from_str(value, "%Y-%m-%d %H:%M:%S%.f")) +} + +pub async fn insert_spend_logs( + executor: impl PgExecutor<'_>, + logs: &[SpendLog], +) -> Result<(), Error> { + if logs.is_empty() { + return Ok(()); + } + let start_times = logs + .iter() + .map(|log| parse_timestamp(&log.start_time)) + .collect::, _>>()?; + let end_times = logs + .iter() + .map(|log| parse_timestamp(&log.end_time)) + .collect::, _>>()?; + let completion_start_times = logs + .iter() + .map(|log| { + log.completion_start_time + .as_deref() + .map(parse_timestamp) + .transpose() + }) + .collect::, _>>()?; + macro_rules! column { + ($field:ident) => { + logs.iter() + .map(|log| log.$field.clone()) + .collect::>() + }; + } + sqlx::query_file!( + "queries/spend/insert_spend_logs.sql", + &column!(request_id), + &column!(call_type), + &column!(api_key), + &column!(spend), + &column!(total_tokens), + &column!(prompt_tokens), + &column!(completion_tokens), + &start_times, + &end_times, + &completion_start_times as &[Option], + &column!(request_duration_ms) as &[Option], + &column!(model), + &column!(model_id), + &column!(model_group), + &column!(custom_llm_provider), + &column!(api_base), + &column!(user), + &column!(metadata), + &column!(cache_hit), + &column!(cache_key), + &column!(team_id), + &column!(organization_id), + &column!(end_user), + &column!(requester_ip_address) as &[Option], + &column!(session_id) as &[Option], + &column!(status), + &column!(litellm_call_id) as &[Option], + ) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{SpendLog, SpendLogRow, insert_spend_logs}; + use crate::test_support::database_pool; + use rstest::{fixture, rstest}; + use serde_json::json; + use sqlx::{Acquire, Connection, PgExecutor, PgPool, types::chrono::NaiveDateTime}; + + #[fixture] + fn logs() -> [SpendLog; 2] { + [ + SpendLog { + request_id: "spend-logs-a".into(), + call_type: "acompletion".into(), + api_key: "key-a".into(), + spend: 0.125, + total_tokens: 17, + prompt_tokens: 11, + completion_tokens: 6, + start_time: "2026-01-02 03:04:05.123".into(), + end_time: "2026-01-02 03:04:06.789".into(), + completion_start_time: Some("2026-01-02 03:04:05.456".into()), + request_duration_ms: Some(1666), + model: "model-a".into(), + model_id: "deployment-a".into(), + model_group: "group-a".into(), + custom_llm_provider: "provider-a".into(), + api_base: "https://a.example.test".into(), + user: "user-a".into(), + metadata: json!({"tenant": "a", "nested": {"tokens": 11}, "flag": false}), + cache_hit: "False".into(), + cache_key: "cache-a".into(), + team_id: "team-a".into(), + organization_id: "org-a".into(), + end_user: "end-user-a".into(), + requester_ip_address: Some("192.0.2.1".into()), + session_id: Some("session-a".into()), + status: "success".into(), + litellm_call_id: Some("call-a".into()), + }, + SpendLog { + request_id: "spend-logs-b".into(), + call_type: "aembedding".into(), + api_key: "key-b".into(), + spend: 2.75, + total_tokens: 43, + prompt_tokens: 29, + completion_tokens: 14, + start_time: "2026-02-03 14:15:16.234".into(), + end_time: "2026-02-03 14:15:18.567".into(), + completion_start_time: None, + request_duration_ms: None, + model: "model-b".into(), + model_id: "deployment-b".into(), + model_group: "group-b".into(), + custom_llm_provider: "provider-b".into(), + api_base: "https://b.example.test".into(), + user: "user-b".into(), + metadata: json!({"tenant": "b", "tags": ["distinct", 7], "nullable": null}), + cache_hit: "True".into(), + cache_key: "cache-b".into(), + team_id: "team-b".into(), + organization_id: "org-b".into(), + end_user: "end-user-b".into(), + requester_ip_address: None, + session_id: None, + status: "failure".into(), + litellm_call_id: None, + }, + ] + } + + async fn read_log( + executor: impl PgExecutor<'_>, + request_id: &str, + ) -> Result { + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + sqlx::query_as::<_, SpendLogRow>( + r#"SELECT * FROM "LiteLLM_SpendLogs" WHERE request_id = $1"#, + ) + .bind(request_id) + .fetch_one(executor) + .await + } + + fn expected_log(log: &SpendLog) -> Result { + Ok(SpendLogRow { + request_id: log.request_id.clone(), + call_type: log.call_type.clone(), + api_key: log.api_key.clone(), + spend: log.spend, + total_tokens: log.total_tokens, + prompt_tokens: log.prompt_tokens, + completion_tokens: log.completion_tokens, + start_time: log.start_time.replace(' ', "T").parse()?, + end_time: log.end_time.replace(' ', "T").parse()?, + completion_start_time: log + .completion_start_time + .as_ref() + .map(|value| value.replace(' ', "T").parse()) + .transpose()?, + request_duration_ms: log.request_duration_ms, + model: log.model.clone(), + model_id: Some(log.model_id.clone()), + model_group: Some(log.model_group.clone()), + custom_llm_provider: Some(log.custom_llm_provider.clone()), + api_base: Some(log.api_base.clone()), + user: Some(log.user.clone()), + metadata: Some(log.metadata.clone()), + cache_hit: Some(log.cache_hit.clone()), + cache_key: Some(log.cache_key.clone()), + request_tags: Some(json!([])), + team_id: Some(log.team_id.clone()), + organization_id: Some(log.organization_id.clone()), + end_user: Some(log.end_user.clone()), + requester_ip_address: log.requester_ip_address.clone(), + messages: Some(json!({})), + response: Some(json!({})), + proxy_server_request: Some(json!({})), + session_id: log.session_id.clone(), + status: Some(log.status.clone()), + litellm_call_id: log.litellm_call_id.clone(), + }) + } + + #[rstest] + #[case::all_columns_and_duplicate()] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn inserts_distinct_rows_without_replacing_duplicates( + logs: [SpendLog; 2], + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + insert_spend_logs(&mut *tx, &logs).await?; + let replacement = SpendLog { + request_id: logs[0].request_id.clone(), + ..logs[1].clone() + }; + insert_spend_logs(&mut *tx, &[replacement]).await?; + assert_eq!( + read_log(&mut *tx, &logs[0].request_id).await?, + expected_log(&logs[0])? + ); + assert_eq!( + read_log(&mut *tx, &logs[1].request_id).await?, + expected_log(&logs[1])? + ); + tx.rollback().await?; + Ok(()) + } + + #[rstest] + #[case::partitioned_duplicate()] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn inserts_into_partitioned_logs_without_replacing_duplicates( + logs: [SpendLog; 2], + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "The test creates a temporary partitioned table at runtime" + )] + sqlx::raw_sql( + r#"CREATE TEMP TABLE "LiteLLM_SpendLogs" ( + LIKE public."LiteLLM_SpendLogs" INCLUDING DEFAULTS INCLUDING GENERATED, + PRIMARY KEY (request_id, "startTime") + ) PARTITION BY RANGE ("startTime"); + CREATE TEMP TABLE spend_logs_default + PARTITION OF pg_temp."LiteLLM_SpendLogs" DEFAULT; + SET LOCAL search_path TO pg_temp, public;"#, + ) + .execute(&mut *tx) + .await?; + tx.clear_cached_statements().await?; + insert_spend_logs(&mut *tx, &logs).await?; + let replacement = SpendLog { + spend: 99.0, + ..logs[0].clone() + }; + insert_spend_logs(&mut *tx, &[replacement]).await?; + #[expect( + clippy::disallowed_methods, + reason = "The partitioned test fixture is created at runtime" + )] + let count: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM pg_temp."LiteLLM_SpendLogs""#) + .fetch_one(&mut *tx) + .await?; + assert_eq!(count.0, 2); + assert_eq!( + read_log(&mut *tx, &logs[0].request_id).await?, + expected_log(&logs[0])? + ); + assert_eq!( + read_log(&mut *tx, &logs[1].request_id).await?, + expected_log(&logs[1])? + ); + let distinct_partition_key = SpendLog { + request_id: logs[0].request_id.clone(), + ..logs[1].clone() + }; + insert_spend_logs(&mut *tx, &[distinct_partition_key]).await?; + #[expect( + clippy::disallowed_methods, + reason = "The partitioned test fixture is created at runtime" + )] + let count: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM pg_temp."LiteLLM_SpendLogs""#) + .fetch_one(&mut *tx) + .await?; + assert_eq!(count.0, 3); + tx.rollback().await?; + Ok(()) + } + + #[rstest] + #[case::positive_offset_across_midnight( + ["2026-01-02T00:04:05.1236+02:00", "2026-01-02T00:04:06.7894+02:00", "2026-01-02T00:04:05.4567+02:00"], + ["2026-01-01T22:04:05.124", "2026-01-01T22:04:06.789", "2026-01-01T22:04:05.457"] + )] + #[case::negative_offset_across_midnight( + ["2026-01-01T23:04:05-02:00", "2026-01-01T23:04:06-02:00", "2026-01-01T23:04:05.456-02:00"], + ["2026-01-02T01:04:05", "2026-01-02T01:04:06", "2026-01-02T01:04:05.456"] + )] + #[case::utc( + ["2026-01-02T03:04:05Z", "2026-01-02T03:04:06Z", "2026-01-02T03:04:05.456Z"], + ["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"] + )] + #[case::naive_iso( + ["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"], + ["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"] + )] + #[case::naive_space( + ["2026-01-02 03:04:05", "2026-01-02 03:04:06", "2026-01-02 03:04:05.456"], + ["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"] + )] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn stores_utc_timestamps( + #[case] input: [&str; 3], + #[case] expected: [&str; 3], + logs: [SpendLog; 2], + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "The test sets a transaction-local timezone to verify timestamp handling" + )] + sqlx::raw_sql("SET LOCAL TIME ZONE 'America/Los_Angeles'") + .execute(&mut *tx) + .await?; + let log = SpendLog { + start_time: input[0].into(), + end_time: input[1].into(), + completion_start_time: Some(input[2].into()), + ..logs[0].clone() + }; + insert_spend_logs(&mut *tx, std::slice::from_ref(&log)).await?; + let row = read_log(&mut *tx, &log.request_id).await?; + assert_eq!(row.start_time, expected[0].parse::()?); + assert_eq!(row.end_time, expected[1].parse::()?); + assert_eq!( + row.completion_start_time, + Some(expected[2].parse::()?) + ); + tx.rollback().await?; + Ok(()) + } + + #[rstest] + #[case::empty_batch()] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn empty_input_leaves_rows_unchanged( + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let before: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM "LiteLLM_SpendLogs""#) + .fetch_one(&mut *tx) + .await?; + insert_spend_logs(&mut *tx, &[]).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let after: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM "LiteLLM_SpendLogs""#) + .fetch_one(&mut *tx) + .await?; + assert_eq!(before, after); + tx.rollback().await?; + Ok(()) + } + + #[rstest] + #[case::invalid_timestamp("not-a-timestamp", "2026-01-02T03:04:06", None)] + #[case::invalid_end("2026-01-02T03:04:05", "not-a-timestamp", None)] + #[case::invalid_completion( + "2026-01-02T03:04:05", + "2026-01-02T03:04:06", + Some("not-a-timestamp") + )] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn invalid_timestamp_allows_caller_rollback( + #[case] start_time: &str, + #[case] end_time: &str, + #[case] completion_start_time: Option<&str>, + logs: [SpendLog; 2], + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut outer = pool.begin().await?; + let mut tx = outer.begin().await?; + let valid = SpendLog { + request_id: "spend-logs-rollback-valid".into(), + ..logs[0].clone() + }; + let invalid = SpendLog { + request_id: "spend-logs-rollback-invalid".into(), + start_time: start_time.into(), + end_time: end_time.into(), + completion_start_time: completion_start_time.map(str::to_owned), + ..logs[1].clone() + }; + insert_spend_logs(&mut *tx, std::slice::from_ref(&valid)).await?; + let result = insert_spend_logs(&mut *tx, &[logs[0].clone(), invalid]).await; + assert!(matches!(result, Err(crate::Error::InvalidTimestamp(_)))); + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let partial_batch: (i64,) = sqlx::query_as( + r#"SELECT count(*) FROM "LiteLLM_SpendLogs" WHERE request_id = ANY($1)"#, + ) + .bind([logs[0].request_id.as_str(), "spend-logs-rollback-invalid"].as_slice()) + .fetch_one(&mut *tx) + .await?; + assert_eq!(partial_batch.0, 0); + tx.rollback().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let remaining: (i64,) = sqlx::query_as( + r#"SELECT count(*) FROM "LiteLLM_SpendLogs" WHERE request_id = ANY($1)"#, + ) + .bind([valid.request_id.as_str(), "spend-logs-rollback-invalid"].as_slice()) + .fetch_one(&mut *outer) + .await?; + assert_eq!(remaining.0, 0); + outer.rollback().await?; + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/team_lock.rs b/litellm-rust/crates/db/src/spend/team_lock.rs new file mode 100644 index 00000000000..7a6b9c7f7d0 --- /dev/null +++ b/litellm-rust/crates/db/src/spend/team_lock.rs @@ -0,0 +1,98 @@ +use sqlx::{PgPool, Postgres, Transaction}; + +use crate::Error; + +pub async fn begin(pool: &PgPool) -> Result, Error> { + let mut transaction = pool.begin().await?; + sqlx::query!("SELECT set_config('lock_timeout', '5000ms', true)") + .fetch_one(&mut *transaction) + .await?; + Ok(transaction) +} + +pub async fn lock_team( + transaction: &mut Transaction<'_, Postgres>, + team_id: &str, +) -> Result<(), Error> { + sqlx::query_file!("queries/spend/lock_team.sql", team_id) + .fetch_one(&mut **transaction) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{begin, lock_team}; + use crate::test_support::database_pool; + use rstest::rstest; + use sqlx::{PgPool, postgres::PgPoolOptions}; + + #[rstest] + #[case::transaction_scoped("team-lock-scope")] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn team_lock_is_held_until_rollback( + #[case] team_id: &str, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let database_pool = database_pool?; + let pool = PgPoolOptions::new() + .max_connections(2) + .connect_with(database_pool.connect_options().as_ref().clone()) + .await?; + let mut first = begin(&pool).await?; + let mut second = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let original_timeout: String = sqlx::query_scalar("SELECT current_setting('lock_timeout')") + .fetch_one(&mut *second) + .await?; + lock_team(&mut first, team_id).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let (backend, timeout): (i32, String) = + sqlx::query_as("SELECT pg_backend_pid(), current_setting('lock_timeout')") + .fetch_one(&mut *first) + .await?; + assert_eq!(timeout, "5s"); + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let held: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(hashtext($1))") + .bind(team_id) + .fetch_one(&mut *second) + .await?; + assert!(!held); + first.rollback().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let released: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(hashtext($1))") + .bind(team_id) + .fetch_one(&mut *second) + .await?; + assert!(released); + let mut next = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let (next_backend, next_timeout): (i32, String) = + sqlx::query_as("SELECT pg_backend_pid(), current_setting('lock_timeout')") + .fetch_one(&mut *next) + .await?; + assert_eq!(next_backend, backend); + assert_eq!(next_timeout, original_timeout); + next.rollback().await?; + second.rollback().await?; + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/team_member_spend.rs b/litellm-rust/crates/db/src/spend/team_member_spend.rs new file mode 100644 index 00000000000..965a39ada4f --- /dev/null +++ b/litellm-rust/crates/db/src/spend/team_member_spend.rs @@ -0,0 +1,162 @@ +use sqlx::{Postgres, Transaction}; + +use super::lock_team; +use crate::Error; + +#[macro_rules_attribute::apply(crate::db_input)] +pub struct TeamMemberSpend { + pub team_id: String, + pub user_id: String, + pub cost: f64, +} + +pub async fn add_team_member_spend( + transaction: &mut Transaction<'_, Postgres>, + members: &[TeamMemberSpend], +) -> Result<(), Error> { + if members.is_empty() { + return Ok(()); + } + let mut members = members.to_vec(); + members.sort_by(|left, right| { + (&left.team_id, &left.user_id).cmp(&(&right.team_id, &right.user_id)) + }); + let mut teams: Vec<&str> = members + .iter() + .map(|member| member.team_id.as_str()) + .collect(); + teams.dedup(); + for team in teams { + lock_team(transaction, team).await?; + } + let rows = serde_json::Value::Array( + members + .iter() + .map(|member| { + serde_json::json!({ + "user_id": member.user_id, + "team_id": member.team_id, + "cost": member.cost, + }) + }) + .collect(), + ); + sqlx::query_file!("queries/spend/upsert_team_member_spend.sql", rows) + .execute(&mut **transaction) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::super::begin; + use super::{TeamMemberSpend, add_team_member_spend}; + use crate::test_support::database_pool; + use rstest::rstest; + use sqlx::PgPool; + + fn members(cost: f64) -> [TeamMemberSpend; 3] { + [ + TeamMemberSpend { + team_id: "member-spend-z".into(), + user_id: "off-roster-existing".into(), + cost: 2.0 * cost, + }, + TeamMemberSpend { + team_id: "member-spend-a".into(), + user_id: "on-roster-missing".into(), + cost, + }, + TeamMemberSpend { + team_id: "member-spend-a".into(), + user_id: "off-roster-missing".into(), + cost: 50.0 * cost, + }, + ] + } + + #[rstest] + #[case::zero_then_quarter(0.25, 0.5, 3.0, 8.0)] + #[case::zero_then_eighth(0.125, 0.25, 2.5, 7.5)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn membership_spend_preserves_roster_guard_and_fractional_costs( + #[case] cost: f64, + #[case] expected_created: f64, + #[case] expected_spend: f64, + #[case] expected_total: f64, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = begin(&pool).await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _teams = sqlx::query( + r#"INSERT INTO "LiteLLM_TeamTable" (team_id, members_with_roles) + VALUES ('member-spend-a', '[{"user_id":"on-roster-missing","role":"user"}]'), + ('member-spend-z', '[]')"#, + ) + .execute(&mut *tx) + .await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _members = sqlx::query( + r#"INSERT INTO "LiteLLM_TeamMembership" (team_id, user_id, spend, total_spend) + VALUES ('member-spend-z', 'off-roster-existing', 2, 7), + ('member-spend-a', 'untouched', 100, 200)"#, + ) + .execute(&mut *tx) + .await?; + add_team_member_spend(&mut tx, &[]).await?; + add_team_member_spend(&mut tx, &members(0.0)).await?; + add_team_member_spend(&mut tx, &members(cost)).await?; + add_team_member_spend(&mut tx, &members(cost)).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, String, f64, f64)> = sqlx::query_as( + r#"SELECT team_id, user_id, spend, total_spend FROM "LiteLLM_TeamMembership" + WHERE team_id IN ('member-spend-a', 'member-spend-z') ORDER BY team_id, user_id"#, + ) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows, + vec![ + ( + "member-spend-a".into(), + "on-roster-missing".into(), + expected_created, + expected_created + ), + ("member-spend-a".into(), "untouched".into(), 100.0, 200.0), + ( + "member-spend-z".into(), + "off-roster-existing".into(), + expected_spend, + expected_total + ), + ] + ); + tx.rollback().await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let remaining: i64 = sqlx::query_scalar( + r#"SELECT COUNT(*) FROM "LiteLLM_TeamMembership" + WHERE team_id IN ('member-spend-a', 'member-spend-z')"#, + ) + .fetch_one(&pool) + .await?; + assert_eq!(remaining, 0); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/team_spend.rs b/litellm-rust/crates/db/src/spend/team_spend.rs new file mode 100644 index 00000000000..a105899bc2a --- /dev/null +++ b/litellm-rust/crates/db/src/spend/team_spend.rs @@ -0,0 +1,182 @@ +use sqlx::PgExecutor; + +use super::EntitySpend; +use crate::Error; + +pub async fn add_team_spend( + executor: impl PgExecutor<'_>, + spend: &EntitySpend, +) -> Result<(), Error> { + spend.validate()?; + if spend.is_empty() { + return Ok(()); + } + sqlx::query_file!("queries/spend/add_team_spend.sql", &spend.ids, &spend.costs) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{EntitySpend, add_team_spend}; + use crate::Error; + use crate::test_support::database_pool; + use rstest::rstest; + use sqlx::PgPool; + + #[rstest] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn duplicate_team_ids_add_every_cost( + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_TeamTable" (team_id, spend, created_at, updated_at) + VALUES ('team-spend-duplicate-a', 2, '2000-01-01', '2000-01-01'), + ('team-spend-duplicate-z', 3, '2000-01-01', '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec![ + "team-spend-duplicate-z".into(), + "team-spend-duplicate-a".into(), + "team-spend-duplicate-z".into(), + "team-spend-duplicate-a".into(), + ], + costs: vec![0.5, 0.25, 1.0, 0.5], + }; + add_team_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, f64)> = sqlx::query_as( + r#"SELECT team_id, spend FROM "LiteLLM_TeamTable" + WHERE team_id = ANY($1) ORDER BY team_id"#, + ) + .bind(&spend.ids) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows, + vec![ + ("team-spend-duplicate-a".into(), 2.75), + ("team-spend-duplicate-z".into(), 4.5), + ] + ); + Ok(()) + } + + #[rstest] + #[case::missing_cost(vec!["team-spend-malformed".into()], vec![])] + #[case::extra_cost(vec!["team-spend-malformed".into()], vec![0.25, 0.5])] + #[case::empty_ids(vec![], vec![0.25])] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn malformed_team_batch_does_not_write( + #[case] ids: Vec, + #[case] costs: Vec, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_TeamTable" (team_id, spend, created_at, updated_at) + VALUES ('team-spend-malformed', 2, '2000-01-01', '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let result = add_team_spend(&mut *tx, &EntitySpend { ids, costs }).await; + assert!(matches!(result, Err(Error::InvalidSpendBatch { .. }))); + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let row: (f64, bool) = sqlx::query_as( + r#"SELECT spend, updated_at = TIMESTAMP '2000-01-01' + FROM "LiteLLM_TeamTable" WHERE team_id = 'team-spend-malformed'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(row, (2.0, true)); + Ok(()) + } + + #[rstest] + #[case::fractional_costs(0.25, 2.5, 4.0)] + #[case::zero_costs(0.0, 2.0, 3.0)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn team_spend_matches_python_update_many( + #[case] cost: f64, + #[case] expected_a_spend: f64, + #[case] expected_z_spend: f64, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_TeamTable" (team_id, spend, created_at, updated_at) + VALUES ('team-spend-a', 2, '2000-01-01', '2000-01-01'), + ('team-spend-z', 3, '2000-01-01', '2000-01-01'), + ('team-spend-untouched', 100, '2000-01-01', '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec![ + "team-spend-z".into(), + "team-spend-a".into(), + "team-spend-missing".into(), + ], + costs: vec![2.0 * cost, cost, 50.0], + }; + add_team_spend(&mut *tx, &EntitySpend::default()).await?; + add_team_spend(&mut *tx, &spend).await?; + add_team_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, f64, bool)> = sqlx::query_as( + r#"SELECT team_id, spend, updated_at > TIMESTAMP '2000-01-01' + FROM "LiteLLM_TeamTable" + WHERE team_id = ANY($1) ORDER BY team_id"#, + ) + .bind(vec![ + "team-spend-a", + "team-spend-z", + "team-spend-missing", + "team-spend-untouched", + ]) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows, + vec![ + ("team-spend-a".into(), expected_a_spend, true), + ("team-spend-untouched".into(), 100.0, false), + ("team-spend-z".into(), expected_z_spend, true), + ] + ); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/spend/user_spend.rs b/litellm-rust/crates/db/src/spend/user_spend.rs new file mode 100644 index 00000000000..2c89c04c1ef --- /dev/null +++ b/litellm-rust/crates/db/src/spend/user_spend.rs @@ -0,0 +1,173 @@ +use sqlx::PgExecutor; + +use super::EntitySpend; +use crate::Error; + +pub async fn add_user_spend( + executor: impl PgExecutor<'_>, + spend: &EntitySpend, +) -> Result<(), Error> { + spend.validate()?; + if spend.is_empty() { + return Ok(()); + } + sqlx::query_file!("queries/spend/add_user_spend.sql", &spend.ids, &spend.costs) + .execute(executor) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + + use super::{EntitySpend, add_user_spend}; + use crate::{Error, test_support::database_pool}; + use rstest::rstest; + use sqlx::PgPool; + + #[rstest] + #[case::fractional_costs(0.25, 2.5, 4.0)] + #[case::zero_costs(0.0, 2.0, 3.0)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn user_spend_matches_python_update_many( + #[case] cost: f64, + #[case] expected_a_spend: f64, + #[case] expected_z_spend: f64, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_UserTable" (user_id, spend, updated_at) + VALUES ('user-spend-a', 2, '2000-01-01'), + ('user-spend-z', 3, '2000-01-01'), + ('user-spend-untouched', 100, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec![ + "user-spend-z".into(), + "user-spend-a".into(), + "user-spend-missing".into(), + ], + costs: vec![2.0 * cost, cost, 50.0], + }; + add_user_spend(&mut *tx, &EntitySpend::default()).await?; + add_user_spend(&mut *tx, &spend).await?; + add_user_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let rows: Vec<(String, f64, bool)> = sqlx::query_as( + r#"SELECT user_id, spend, updated_at > TIMESTAMP '2000-01-01' + FROM "LiteLLM_UserTable" + WHERE user_id = ANY($1) ORDER BY user_id"#, + ) + .bind(vec![ + "user-spend-a", + "user-spend-z", + "user-spend-missing", + "user-spend-untouched", + ]) + .fetch_all(&mut *tx) + .await?; + assert_eq!( + rows, + vec![ + ("user-spend-a".into(), expected_a_spend, true), + ("user-spend-untouched".into(), 100.0, false), + ("user-spend-z".into(), expected_z_spend, true), + ] + ); + Ok(()) + } + + #[rstest] + #[case::direct_duplicate_batch(vec![0.25, 0.75], 3.0)] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn user_spend_sums_duplicate_ids( + #[case] costs: Vec, + #[case] expected_spend: f64, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_UserTable" (user_id, spend, updated_at) + VALUES ('user-spend-duplicate', 2, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec!["user-spend-duplicate".into(), "user-spend-duplicate".into()], + costs, + }; + add_user_spend(&mut *tx, &spend).await?; + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let row: (f64, bool) = sqlx::query_as( + r#"SELECT spend, updated_at > TIMESTAMP '2000-01-01' + FROM "LiteLLM_UserTable" WHERE user_id = 'user-spend-duplicate'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(row, (expected_spend, true)); + Ok(()) + } + + #[rstest] + #[case::missing_cost(vec![])] + #[case::extra_cost(vec![0.25, 0.75])] + #[tokio::test] + #[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"] + async fn user_spend_rejects_malformed_batch_without_writes( + #[case] costs: Vec, + #[future(awt)] database_pool: Result>, + ) -> Result<(), Box> { + let pool = database_pool?; + let mut tx = pool.begin().await?; + #[expect( + clippy::disallowed_methods, + reason = "Fixture writes are outside the compile-checked query set" + )] + let _inserted = sqlx::query( + r#"INSERT INTO "LiteLLM_UserTable" (user_id, spend, updated_at) + VALUES ('user-spend-malformed', 2, '2000-01-01')"#, + ) + .execute(&mut *tx) + .await?; + let spend = EntitySpend { + ids: vec!["user-spend-malformed".into()], + costs, + }; + let result = add_user_spend(&mut *tx, &spend).await; + assert!(matches!(result, Err(Error::InvalidSpendBatch { .. }))); + #[expect( + clippy::disallowed_methods, + reason = "Test assertions are outside the compile-checked query set" + )] + let row: (f64, bool) = sqlx::query_as( + r#"SELECT spend, updated_at = TIMESTAMP '2000-01-01' + FROM "LiteLLM_UserTable" WHERE user_id = 'user-spend-malformed'"#, + ) + .fetch_one(&mut *tx) + .await?; + assert_eq!(row, (2.0, true)); + Ok(()) + } +} diff --git a/litellm-rust/crates/db/src/test_support.rs b/litellm-rust/crates/db/src/test_support.rs new file mode 100644 index 00000000000..992be4a4447 --- /dev/null +++ b/litellm-rust/crates/db/src/test_support.rs @@ -0,0 +1,10 @@ +use std::error::Error as StdError; + +use rstest::fixture; +use sqlx::PgPool; + +#[fixture] +pub async fn database_pool() -> Result> { + let database_url = std::env::var("DATABASE_URL")?; + Ok(PgPool::connect(&database_url).await?) +} diff --git a/litellm-rust/crates/gateway/AGENTS.md b/litellm-rust/crates/gateway/AGENTS.md index d54fa062f2b..a33bd19cc80 100644 --- a/litellm-rust/crates/gateway/AGENTS.md +++ b/litellm-rust/crates/gateway/AGENTS.md @@ -3,3 +3,10 @@ - Endpoint paths, request handling, model resolution, and response encoding belong to the mounted crates. Provider execution belongs to the `inference-` crates and `llms` - Inject shared state and infrastructure. Avoid global runtimes, duplicate client pools, and abstractions for hypothetical endpoint groups - Test mounting and server lifecycle through public HTTP behavior. Test endpoint semantics in the owning crate + +## Redis +- `build_redis` builds the Redis clients once at boot (`src/redis.rs`). Components get a clone as a field, never a per-request extractor or a pool. redis-rs multiplexes one connection per node, so there is no connection pool and pool settings like `max_connections` are accepted and ignored +- Which Redis backs which role mirrors the proxy: coordination is `general_settings.coordination_redis`, else the `cache_params` Redis when `cache: true` and `type: redis`, else `REDIS_*` env. The transaction buffer reuses coordination, else env when `use_redis_transaction_buffer` is on. The response cache gets its own client so large values never queue in front of coordination commands +- `litellm-config` follows `litellm/_redis.py`'s merge of config blocks and `REDIS_*` env. A redis-py setting we can't honor (sentinel, IAM, custom TLS files, ...) fails boot with its name, never silently dropped +- Boot never waits for a configured Redis (clients connect lazily). Only the env-only coordination fallback is pinged, for 2s, and treated as absent when unreachable, as in Python +- `/health/readiness` never touches Redis, matching the proxy diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 387bcd43231..b2a2f09bfc7 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -19,6 +19,8 @@ litellm-gateway-auth.workspace = true litellm-gateway-ui.workspace = true litellm-auth-types.workspace = true litellm-config.workspace = true +litellm-circuit-breaker.workspace = true +litellm-redis-client.workspace = true litellm-http = { workspace = true, features = ["mcp"] } litellm-llms.workspace = true litellm-secrets.workspace = true @@ -31,6 +33,7 @@ uuid.workspace = true tower-sessions-moka-store = "0.15.0" [dev-dependencies] +litellm-redis-testing.workspace = true base64.workspace = true rstest.workspace = true tempfile.workspace = true diff --git a/litellm-rust/crates/gateway/src/error.rs b/litellm-rust/crates/gateway/src/error.rs index 75dc04d179d..170a8284c18 100644 --- a/litellm-rust/crates/gateway/src/error.rs +++ b/litellm-rust/crates/gateway/src/error.rs @@ -8,6 +8,15 @@ pub enum Error { McpSetting(String), #[error(transparent)] Mcp(#[from] litellm_gateway_mcp::ConnectError), + #[error(transparent)] + Config(#[from] litellm_config::Error), + #[error("invalid Redis setting {setting}: {reason}")] + RedisInvalid { + setting: &'static str, + reason: &'static str, + }, + #[error(transparent)] + RedisConnect(#[from] litellm_redis_client::ConnectError), #[error("MCP requires a configured master key")] Auth(#[from] litellm_gateway_auth::Error), } diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 0b3aba81181..2ed2fc80a51 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,7 +1,9 @@ mod auth; mod error; +mod redis; mod secrets; pub use error::Error; +pub use redis::RedisClients; use std::{sync::Arc, time::Instant}; @@ -70,6 +72,17 @@ pub fn build_inference(config: &Config) -> Result, Error> { )?)) } +pub async fn build_redis(config: &Config) -> Result { + let environment = secrets::environment_values(&config.environment_variables)?; + let lookup = |name: &str| { + environment + .get(name) + .map(|value| value.expose().to_owned()) + .or_else(|| std::env::var(name).ok()) + }; + redis::connect(config, &lookup).await +} + pub async fn build_mcp( config: &Config, secrets: Arc, @@ -128,12 +141,17 @@ pub fn router( litellm_gateway_auth::authenticate, )) .layer(axum::middleware::from_fn(log_request)); + let health = Router::new().route("/health/readiness", axum::routing::get(readiness)); match ui { - Some(ui) => inference.merge(ui), - None => inference, + Some(ui) => inference.merge(ui).merge(health), + None => inference.merge(health), } } +async fn readiness() -> axum::Json { + axum::Json(serde_json::json!({"status": "healthy", "db": "Not connected"})) +} + async fn log_request(request: Request, next: Next) -> Response { let request_id = Uuid::new_v4().to_string(); let log_body_chunks = tracing::enabled!(tracing::Level::DEBUG); diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index f8150e6e860..47c1fe83f50 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -46,6 +46,7 @@ async fn main() -> Result<(), Box> { Logger::new(StderrSink { level }).install_global()?; let config = Config::load(settings.litellm_config)?; let inference = litellm_gateway::build_inference(&config)?; + let _redis = litellm_gateway::build_redis(&config).await?; let ui = match std::env::var_os("LITELLM_UI_PATH") { Some(directory) => { let ui_settings: UiSettings = settings::from_iter(std::env::vars_os())?; diff --git a/litellm-rust/crates/gateway/src/redis.rs b/litellm-rust/crates/gateway/src/redis.rs new file mode 100644 index 00000000000..197f40e7988 --- /dev/null +++ b/litellm-rust/crates/gateway/src/redis.rs @@ -0,0 +1,577 @@ +use std::time::Duration; + +use litellm_circuit_breaker::{Breaker, Config as BreakerConfig, State}; +use litellm_config::{ + CacheSettings, Config, Lookup, RedisEndpoint, RedisSettings, ResolvedRedis, Value, + environment_has_target, +}; +use litellm_redis_client::{ + ClusterEndpoint, ClusterNode, DEFAULT_TIMEOUT, Endpoint, HostEndpoint, Namespace, RedisClient, + RedisConfig, +}; + +use crate::Error; + +const ENVIRONMENT_PING_TIMEOUT: Duration = Duration::from_secs(2); +type Params = ResolvedRedis; + +pub struct RedisClients { + pub coordination: Option, + pub transaction_buffer: Option, + pub response_cache: Option, +} + +#[derive(Debug, PartialEq)] +pub(crate) struct Plan { + coordination: Option, + transaction_buffer_fallback: Option, + response_cache: Option, +} + +#[derive(Debug, PartialEq)] +enum Target { + Configured(Params), + Environment(Params), +} + +pub(crate) fn plan(config: &Config, lookup: Lookup<'_>) -> Result { + let response_cache = redis_cache_params(config) + .map(|settings| settings.resolve_cache(lookup)) + .transpose()?; + let coordination = match &config.general_settings.coordination_redis { + Some(settings) => Some(Target::Configured(settings.resolve(lookup)?)), + None => match (&response_cache, config.litellm_settings.cache == Some(true)) { + (Some(cache), _) => Some(Target::Configured(cache.clone())), + (None, true) => environment(lookup)?.map(Target::Configured), + (None, false) => optional_environment(lookup).map(Target::Environment), + }, + }; + let transaction_buffer_fallback = match (&coordination, use_transaction_buffer(config)) { + (Some(Target::Configured(_)), _) | (_, false) => None, + (Some(Target::Environment(_)) | None, true) => environment(lookup)?, + }; + Ok(Plan { + coordination, + transaction_buffer_fallback, + response_cache, + }) +} + +pub async fn connect(config: &Config, lookup: Lookup<'_>) -> Result { + let plan = plan(config, lookup)?; + let breaker = breaker_config(lookup)?; + let coordination = match plan.coordination { + Some(Target::Configured(params)) => Some(client(params, breaker)?), + Some(Target::Environment(params)) => match client(params, breaker) { + Ok(client) => reachable(client).await, + Err(_) => { + tracing::warn!( + "REDIS_* environment variables did not form a valid Redis client, cross-pod coordination stays in memory" + ); + None + } + }, + None => None, + }; + let transaction_buffer = match (&coordination, plan.transaction_buffer_fallback) { + (Some(coordination), _) => Some(coordination.clone()), + (None, Some(params)) => Some(client(params, breaker)?), + (None, None) => None, + }; + let response_cache = plan + .response_cache + .map(|params| client(params, breaker)) + .transpose()?; + Ok(RedisClients { + coordination, + transaction_buffer, + response_cache, + }) +} + +fn redis_cache_params(config: &Config) -> Option<&RedisSettings> { + let settings = &config.litellm_settings; + let params = settings.cache_params.as_ref()?; + match (settings.cache, params) { + (Some(true), CacheSettings::Redis(redis)) => Some(redis), + _ => None, + } +} + +fn environment(lookup: Lookup<'_>) -> Result, Error> { + match environment_has_target(lookup) { + true => RedisSettings::default() + .resolve(lookup) + .map(Some) + .map_err(Into::into), + false => Ok(None), + } +} + +fn optional_environment(lookup: Lookup<'_>) -> Option { + match environment(lookup) { + Ok(params) => params, + Err(error) => { + tracing::warn!(error = %error, "REDIS_* environment variables are invalid, cross-pod coordination stays in memory"); + None + } + } +} + +fn use_transaction_buffer(config: &Config) -> bool { + match config + .general_settings + .additional_fields + .get("use_redis_transaction_buffer") + { + Some(Value::Bool(enabled)) => *enabled, + Some(Value::String(text)) => matches!( + text.to_ascii_lowercase().as_str(), + "true" | "1" | "yes" | "on" + ), + _ => false, + } +} + +async fn reachable(client: RedisClient) -> Option { + match tokio::time::timeout(ENVIRONMENT_PING_TIMEOUT, client.ping()).await { + Ok(Ok(())) => Some(client), + Ok(Err(error)) => { + tracing::warn!(%error, "REDIS_* environment variables named a Redis that is not reachable, cross-pod coordination stays in memory"); + None + } + Err(_) => { + tracing::warn!( + "REDIS_* environment variables named a Redis that did not answer PING in 2s, cross-pod coordination stays in memory" + ); + None + } + } +} + +fn client(params: Params, breaker: Option) -> Result { + tracing::info!(endpoint = ?params.endpoint, namespace = ?params.namespace, "redis client configured"); + let endpoint = match params.endpoint { + RedisEndpoint::Url(url) => Endpoint::Url(url.expose().to_owned()), + RedisEndpoint::Host(host) => Endpoint::Host(HostEndpoint { + host: host.host, + port: host.port, + username: host.username.map(|value| value.expose().to_owned()), + password: host.password.map(|value| value.expose().to_owned()), + tls: host.tls, + db: host.db, + }), + RedisEndpoint::Cluster(cluster) => Endpoint::Cluster(ClusterEndpoint { + nodes: cluster + .nodes + .into_iter() + .map(|node| ClusterNode { + host: node.host, + port: node.port, + }) + .collect(), + username: cluster.username.map(|value| value.expose().to_owned()), + password: cluster.password.map(|value| value.expose().to_owned()), + tls: cluster.tls, + }), + }; + Ok(RedisClient::new(RedisConfig { + endpoint, + namespace: params.namespace.map(Namespace::new).unwrap_or_default(), + timeout: params.timeout.unwrap_or(DEFAULT_TIMEOUT), + breaker: breaker.map(|config| Breaker::new(config, Box::new(log_transition))), + })?) +} + +fn log_transition(transition: &litellm_circuit_breaker::Transition) { + match transition.to { + State::Open { .. } => tracing::warn!( + generation = transition.generation, + "redis circuit breaker opened, fast-failing redis calls" + ), + State::HalfOpen => tracing::info!( + generation = transition.generation, + "redis circuit breaker half open, probing redis" + ), + State::Closed { .. } => tracing::info!( + generation = transition.generation, + "redis circuit breaker closed, redis recovered" + ), + } +} + +fn breaker_config(lookup: Lookup<'_>) -> Result, Error> { + let enabled = lookup("REDIS_CIRCUIT_BREAKER_ENABLED") + .is_none_or(|value| value.eq_ignore_ascii_case("true")); + if !enabled { + return Ok(None); + } + let defaults = BreakerConfig::default(); + let seconds = |name: &'static str, default: Duration| -> Result { + lookup(name) + .map(|value| { + value + .parse::() + .ok() + .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) + .ok_or(Error::RedisInvalid { + setting: name, + reason: "must be a non-negative number of seconds", + }) + }) + .unwrap_or(Ok(default)) + }; + let failure_threshold = lookup("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD") + .map(|value| { + value.parse().map_err(|_| Error::RedisInvalid { + setting: "REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", + reason: "must be an integer", + }) + }) + .unwrap_or(Ok(defaults.failure_threshold))?; + Ok(Some(BreakerConfig { + failure_threshold, + recovery_timeout: seconds( + "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", + defaults.recovery_timeout, + )?, + timeout_min_duration: seconds( + "REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", + defaults.timeout_min_duration, + )?, + })) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, time::Duration}; + + use litellm_config::{Config, RedisEndpoint, RedisHost}; + use litellm_redis_testing::RedisServer; + use rstest::rstest; + + use super::{Params, Plan, Target, connect, plan}; + use crate::Error; + + fn host(name: &str, port: u16) -> Params { + Params { + endpoint: RedisEndpoint::Host(RedisHost { + host: name.to_owned(), + port, + username: None, + password: None, + tls: false, + db: 0, + }), + namespace: None, + timeout: None, + } + } + + fn resolved(yaml: &str, environment: &[(&str, &str)]) -> Result { + let environment: HashMap = environment + .iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(); + plan(&Config::from_yaml(yaml).unwrap(), &|name| { + environment.get(name).cloned() + }) + } + + fn configured(coordination: Params, response_cache: Option) -> Plan { + Plan { + coordination: Some(Target::Configured(coordination)), + transaction_buffer_fallback: None, + response_cache, + } + } + + const NO_REDIS: Plan = Plan { + coordination: None, + transaction_buffer_fallback: None, + response_cache: None, + }; + + #[rstest] + #[case::nothing_configured("model_list: []", &[], NO_REDIS)] + #[case::coordination_block_wins_over_the_cache_redis( + "general_settings:\n coordination_redis:\n host: coord\nlitellm_settings:\n cache: true\n cache_params:\n type: redis\n host: cache", + &[], + configured(host("coord", 6379), Some(host("cache", 6379))) + )] + #[case::cache_params_without_cache_true_are_unused( + "litellm_settings:\n cache_params:\n type: redis\n host: cache", + &[], + NO_REDIS + )] + #[case::non_redis_cache_takes_coordination_from_the_environment_without_a_ping( + "litellm_settings:\n cache: true\n cache_params:\n type: local", + &[("REDIS_HOST", "env")], + configured(host("env", 6379), None) + )] + #[case::environment_only_coordination_is_probed( + "model_list: []", + &[("REDIS_HOST", "env")], + Plan { coordination: Some(Target::Environment(host("env", 6379))), transaction_buffer_fallback: None, response_cache: None } + )] + #[case::the_transaction_buffer_falls_back_to_the_environment_only_when_enabled( + "general_settings:\n use_redis_transaction_buffer: \"true\"", + &[("REDIS_HOST", "env")], + Plan { coordination: Some(Target::Environment(host("env", 6379))), transaction_buffer_fallback: Some(host("env", 6379)), response_cache: None } + )] + fn resolves_each_redis_role_like_the_proxy( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] expected: Plan, + ) { + assert_eq!(resolved(yaml, environment).unwrap(), expected); + } + + #[rstest] + fn redis_migration_regression_preserves_borrowed_cache_namespace() { + let yaml = "litellm_settings:\n cache: true\n cache_params:\n type: redis\n host: cache\n namespace: responses\n ttl: 600"; + let cache = Params { + namespace: Some("responses".to_owned()), + ..host("cache", 6379) + }; + + assert_eq!( + resolved(yaml, &[]).unwrap(), + configured(cache.clone(), Some(cache)) + ); + } + + #[rstest] + #[tokio::test] + async fn borrowed_coordination_and_transaction_buffer_share_the_cache_namespace() { + let server = RedisServer::start().await.unwrap(); + let config = Config::from_yaml(&format!( + "litellm_settings:\n cache: true\n cache_params:\n type: redis\n host: 127.0.0.1\n port: {}\n namespace: responses", + port_of(&server) + )) + .unwrap(); + let clients = connect(&config, &|_| None).await.unwrap(); + let coordination = clients.coordination.unwrap(); + let buffer = clients.transaction_buffer.unwrap(); + let cache = clients.response_cache.unwrap(); + let unnamespaced = connect(&coordination_config(&server), &|_| None) + .await + .unwrap() + .coordination + .unwrap(); + + coordination + .set("rate-limit", "1", Duration::from_secs(60)) + .await + .unwrap(); + buffer + .set("spend", "2", Duration::from_secs(60)) + .await + .unwrap(); + + assert_eq!( + cache.get::("rate-limit").await.unwrap().as_deref(), + Some("1") + ); + assert_eq!( + cache.get::("spend").await.unwrap().as_deref(), + Some("2") + ); + assert_eq!( + unnamespaced + .get::("responses:rate-limit") + .await + .unwrap() + .as_deref(), + Some("1") + ); + assert_eq!( + unnamespaced + .get::("responses:spend") + .await + .unwrap() + .as_deref(), + Some("2") + ); + assert!( + unnamespaced + .get::("rate-limit") + .await + .unwrap() + .is_none() + ); + assert!(unnamespaced.get::("spend").await.unwrap().is_none()); + } + + #[rstest] + #[case::malformed_port(&[("REDIS_HOST", "env"), ("REDIS_PORT", "not-a-port")])] + #[case::malformed_cluster_nodes(&[("REDIS_CLUSTER_NODES", "[{\"host\": ")])] + fn redis_migration_regression_malformed_optional_environment_is_absent( + #[case] environment: &[(&str, &str)], + ) { + assert_eq!(resolved("model_list: []", environment).unwrap(), NO_REDIS); + } + + #[rstest] + fn redis_migration_regression_explicit_malformed_setting_still_fails() { + let error = resolved( + "general_settings:\n coordination_redis:\n host: coord\n port: not-a-port", + &[], + ) + .unwrap_err(); + + assert!(matches!( + error, + Error::Config(litellm_config::Error::RedisInvalid { + setting: "port", + .. + }) + )); + } + + #[rstest] + fn redis_migration_regression_enabled_transaction_buffer_rejects_invalid_environment() { + let error = resolved( + "general_settings:\n use_redis_transaction_buffer: true", + &[("REDIS_HOST", "env"), ("REDIS_PORT", "not-a-port")], + ) + .unwrap_err(); + + assert!(matches!( + error, + Error::Config(litellm_config::Error::RedisInvalid { + setting: "port", + .. + }) + )); + } + + #[rstest] + fn redis_migration_regression_invalid_response_cache_still_fails() { + let error = resolved( + "litellm_settings:\n cache: true\n cache_params:\n type: redis\n host: cache\n port: not-a-port", + &[], + ) + .unwrap_err(); + + assert!(matches!( + error, + Error::Config(litellm_config::Error::RedisInvalid { + setting: "port", + .. + }) + )); + } + + #[rstest] + #[tokio::test] + async fn redis_migration_regression_invalid_optional_environment_url_is_absent() { + let config = Config::from_yaml("model_list: []").unwrap(); + + let clients = connect(&config, &|name| { + (name == "REDIS_URL").then(|| "not a redis URL".to_owned()) + }) + .await + .unwrap(); + + assert!(clients.coordination.is_none()); + } + + fn port_of(server: &RedisServer) -> String { + server.url().rsplit(':').next().unwrap().to_owned() + } + + fn coordination_config(server: &RedisServer) -> Config { + Config::from_yaml(&format!( + "general_settings:\n coordination_redis:\n host: 127.0.0.1\n port: {}\n socket_timeout: 0.5", + port_of(server) + )) + .unwrap() + } + + async fn eventually_pings(client: &litellm_redis_client::RedisClient) -> bool { + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while tokio::time::Instant::now() < deadline { + if client.ping().await.is_ok() { + return true; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + false + } + + #[rstest] + #[tokio::test] + async fn boot_does_not_wait_for_a_configured_redis_that_is_down() { + let server = RedisServer::start().await.unwrap(); + server.stop().await.unwrap(); + + let clients = connect(&coordination_config(&server), &|_| None) + .await + .unwrap(); + let coordination = clients.coordination.unwrap(); + let while_down = coordination.ping().await; + server.resume().await.unwrap(); + + assert!(while_down.is_err()); + assert!(eventually_pings(&coordination).await); + } + + #[rstest] + #[tokio::test] + async fn the_coordination_client_survives_a_redis_restart_and_backs_the_transaction_buffer() { + let server = RedisServer::start().await.unwrap(); + let clients = connect(&coordination_config(&server), &|_| None) + .await + .unwrap(); + let coordination = clients.coordination.unwrap(); + coordination + .set("written", "1", Duration::from_secs(60)) + .await + .unwrap(); + + server.restart().await.unwrap(); + + assert!(eventually_pings(&coordination).await); + let buffer = clients.transaction_buffer.unwrap(); + assert!(buffer.ping().await.is_ok()); + } + + #[rstest] + #[tokio::test] + async fn an_unreachable_environment_only_redis_leaves_coordination_in_memory() { + let server = RedisServer::start().await.unwrap(); + let port = port_of(&server); + server.stop().await.unwrap(); + let config = Config::from_yaml("model_list: []").unwrap(); + + let clients = connect(&config, &|name| match name { + "REDIS_HOST" => Some("127.0.0.1".to_owned()), + "REDIS_PORT" => Some(port.clone()), + _ => None, + }) + .await + .unwrap(); + + assert!(clients.coordination.is_none()); + assert!(clients.transaction_buffer.is_none()); + } + + #[rstest] + #[tokio::test] + async fn a_reachable_environment_only_redis_becomes_coordination() { + let server = RedisServer::start().await.unwrap(); + let port = port_of(&server); + let config = Config::from_yaml("model_list: []").unwrap(); + + let clients = connect(&config, &|name| match name { + "REDIS_HOST" => Some("127.0.0.1".to_owned()), + "REDIS_PORT" => Some(port.clone()), + _ => None, + }) + .await + .unwrap(); + + assert!(clients.coordination.unwrap().ping().await.is_ok()); + } +} diff --git a/litellm-rust/crates/gateway/tests/server.rs b/litellm-rust/crates/gateway/tests/server.rs index 71edd3c3650..53a97e7098c 100644 --- a/litellm-rust/crates/gateway/tests/server.rs +++ b/litellm-rust/crates/gateway/tests/server.rs @@ -287,3 +287,33 @@ async fn ui_routes_are_absent_when_not_mounted( .unwrap(); assert_eq!(response.status().as_u16(), 404); } + +#[rstest] +#[case::with_master_key("model_list: []\ngeneral_settings:\n master_key: gateway-key\n")] +#[case::without_master_key("model_list: []\n")] +#[tokio::test] +async fn readiness_answers_without_credentials_like_the_proxy_with_no_database( + inference: Arc, + #[case] yaml: &str, +) { + let config = Config::from_yaml(yaml).unwrap(); + let app = litellm_gateway::router(inference, &config, None, None); + + let response = app + .oneshot( + Request::get("/health/readiness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status().as_u16(), 200); + let body = axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(); + assert_eq!( + serde_json::from_slice::(&body).unwrap(), + json!({"status": "healthy", "db": "Not connected"}) + ); +} diff --git a/litellm-rust/crates/inference-messages/src/common_utils.rs b/litellm-rust/crates/inference-messages/src/common_utils.rs index 7394d7ae21e..9be7ee6ffc3 100644 --- a/litellm-rust/crates/inference-messages/src/common_utils.rs +++ b/litellm-rust/crates/inference-messages/src/common_utils.rs @@ -6,7 +6,7 @@ use litellm_llms::{ anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG, azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG, base_llm::messages::transformation::BaseMessagesConfig, - bedrock::messages::invoke_transformations::anthropic_claude3_transformation::BEDROCK_ANTHROPIC_MESSAGES_CONFIG, + bedrock::messages::invoke_transformations::anthropic_claude_transformation::BEDROCK_ANTHROPIC_MESSAGES_CONFIG, deepseek::messages::transformation::DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG, edenai::messages::transformation::EDENAI_MESSAGES_CONFIG, minimax::messages::transformation::MINIMAX_MESSAGES_CONFIG, diff --git a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs similarity index 100% rename from litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs rename to litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs diff --git a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/mod.rs b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/mod.rs index 4d67a0c0696..68ceaac3444 100644 --- a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/mod.rs +++ b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/mod.rs @@ -1 +1 @@ -pub mod anthropic_claude3_transformation; +pub mod anthropic_claude_transformation; diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 6fc05c43146..8afde8362f6 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -53,7 +53,7 @@ pyo3.workspace = true prost.workspace = true pyo3-async-runtimes.workspace = true reqwest.workspace = true -redis = { version = "1.7.0", features = ["tls-rustls"] } +redis = { workspace = true, features = ["tls-rustls"] } serde_json.workspace = true strum.workspace = true veil.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/secrets/provider.rs b/litellm-rust/crates/python-bridge/src/secrets/provider.rs index 568a0cd2228..5f4cd193fcf 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/provider.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/provider.rs @@ -88,20 +88,23 @@ fn aws_context( } fn read_timeout(value: Option<&Bound<'_, PyAny>>) -> PyResult> { - let Some(value) = value.filter(|value| !value.is_none()) else { - return Ok(None); - }; - let seconds = match value.extract::() { - Ok(value) => Some(value), - Err(_) => value.getattr("read")?.extract::>()?, - }; - seconds + read_timeout_seconds(value)? .map(|value| { Duration::try_from_secs_f64(value).map_err(|_| PyValueError::new_err("invalid timeout")) }) .transpose() } +pub(super) fn read_timeout_seconds(value: Option<&Bound<'_, PyAny>>) -> PyResult> { + let Some(value) = value.filter(|value| !value.is_none()) else { + return Ok(None); + }; + match value.extract::() { + Ok(value) => Ok(Some(value)), + Err(_) => value.getattr("read")?.extract::>(), + } +} + fn vault_context(params: Option<&Bound<'_, PyAny>>) -> PyResult { let params = params.and_then(|value| value.cast::().ok()); let nested = params diff --git a/litellm-rust/crates/python-bridge/src/secrets/vault.rs b/litellm-rust/crates/python-bridge/src/secrets/vault.rs index 46b29839fe4..79281662848 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/vault.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/vault.rs @@ -2,10 +2,12 @@ mod operation; pub(super) use operation::{Failure, FailureKind, FailureStage, delete, rotate, write}; +use litellm_core_utils::exception_mapping_utils::{TimeoutPhase, timeout_message}; use litellm_secrets::hashicorp::{Error, RawOperationError}; use pyo3::prelude::*; use super::mutation::{error_value, http_message, json_value}; +use super::provider::read_timeout_seconds; const HTTP_TRANSPORT_MODULE: &str = "litellm.rust_bridge.host.http_transport"; @@ -69,21 +71,17 @@ fn cause_message(py: Python<'_>, kind: FailureKind, context: &ErrorContext) -> P } FailureKind::Native(RawOperationError::Timeout { method, elapsed }) => { if method == "POST" { - let elapsed = py - .import("builtins")? - .call_method1("round", (elapsed.as_secs_f64(), 3))?; let kwargs = pyo3::types::PyDict::new(py); kwargs.set_item( "message", - format!( - "Connection timed out. Timeout passed={}, time taken={} seconds", - context.timeout.as_deref().unwrap_or("None"), - elapsed.str()? + timeout_message( + TimeoutPhase::Read, + context.timeout, + Some(elapsed.as_secs_f64()), ), )?; kwargs.set_item("model", "default-model-name")?; kwargs.set_item("llm_provider", "litellm-httpx-handler")?; - kwargs.set_item("headers", pyo3::types::PyDict::new(py))?; py.import("litellm.exceptions")? .getattr("Timeout")? .call((), Some(&kwargs))? @@ -159,7 +157,7 @@ fn response_text(py: Python<'_>, body: &[u8]) -> PyResult { #[derive(Default)] pub(super) struct ErrorContext { - timeout: Option, + timeout: Option, aiohttp: bool, } @@ -173,7 +171,7 @@ impl ErrorContext { return Ok(Self::default()); } Ok(Self { - timeout: timeout.map(|value| value.str()?.extract()).transpose()?, + timeout: read_timeout_seconds(timeout)?, aiohttp: py .import(HTTP_TRANSPORT_MODULE)? .getattr("uses_aiohttp_transport")? diff --git a/litellm-rust/crates/redis-client/AGENTS.md b/litellm-rust/crates/redis-client/AGENTS.md new file mode 100644 index 00000000000..c73a4ed8c4d --- /dev/null +++ b/litellm-rust/crates/redis-client/AGENTS.md @@ -0,0 +1,43 @@ +- The one async Redis client for the gateway and the Rust cache crates. It owns connecting, timeouts, error classes and the key namespace +- It knows nothing about domains, so key builders, TTL policy, value formats and fallback decisions stay in the crate that owns the state. Every write takes its TTL from the caller, and there is no default +- Async only. Build one `RedisClient` per Redis instance at the process boundary and clone it. Never add a pool, a sync API, `spawn_blocking`, or a client built inside a route +- Set every redis-rs timeout and retry explicitly. Its defaults (500ms response, 1s connect, 6 reconnect retries) are not the Python proxy's (5s socket timeout from `REDIS_SOCKET_TIMEOUT`) +- Resolving config from `config.yaml` / `REDIS_*` env the way `litellm/_redis.py` does belongs to the consumer that reads that config, not here. `RedisConfig` takes already-resolved values + +## Errors +- `Transport`, `Server`, `Decode`, each wrapping the redis-rs error as `#[source]`. `ConnectError` is separate because only `connect` returns it +- Classify in `From for Error` by `ErrorKind`. LOADING, CLUSTERDOWN, MASTERDOWN, TRYAGAIN and READONLY are typed server replies, but they mean the node can't serve right now, so they are `Transport`. Codes redis-rs doesn't type (WRONGTYPE) come back as `ErrorKind::Extension` and are `Server`. Each code gets its own rstest case built from a parsed RESP error +- A `Server` error never drops the connection +- `Transport` includes timeouts, where the command may still have applied. Callers of counters and locks treat it as an unknown outcome +- Never swallow an error or turn it into a miss. The caller picks the fallback +- Reads return `Option`: a missing key is `None`, and a value that doesn't decode to `T` (`"15.0"` as `i64`) is `Decode` + +## Circuit breaker +- The `litellm-circuit-breaker` `Breaker` is injected through `RedisConfig::breaker` and shared by every clone. `None` turns it off +- Every public call (each command, `eval`, one `run` of a pipeline) takes one admission through `guarded` / `admit` in `client.rs`. Never call the connection around it +- `Transport` counts as a failure (`Timeout` when `is_timeout()`, else `Hard`). `Server` and `Decode` count as nothing, since Redis answered and the request was bad. A probe that gets one still reopens, through the dropped-probe rule in `circuit-breaker`. A pipeline counts one failure if any op came back `Transport`, else one success +- An open breaker returns `Error::BreakerOpen` without touching the socket +- Python doesn't count MASTERDOWN, TRYAGAIN or READONLY as health failures. Here they are `Transport` and do count + +## Namespace +- Applied once, here, to every key. A key that already starts with `{namespace}:` is left alone, matching `RedisCache.check_and_fix_namespace` +- The rule never changes, because Python and Rust pods share keys in the same Redis + +## Commands +- One typed method per command, built with `redis::cmd`. Empty batches (MGET, DEL) return without a round trip, since Redis rejects an empty MGET +- On cluster, redis-rs splits MGET and DEL by slot and returns MGET values in input order. Other multi-key commands are not split, so a new one needs a cross-slot test +- TTLs are `Duration` and are sent in milliseconds (`PX`, `PEXPIRE`, `PTTL`) + +## Scripts and pipelines +- A `Script` declares every key it touches in KEYS. On cluster an undeclared key works or fails depending on which node owns its slot, so it can pass tests and break in production +- `eval` goes through redis-rs `invoke_async`, which loads the script on NOSCRIPT. Inside a pipeline nothing reloads, so a NOSCRIPT op reruns alone with `invoke_async` +- A pipeline is never atomic. Every op gets its own `Result`, from `.ignore_errors()`, so one rejected op leaves the others alone. A transport failure fails every op of that round trip +- redis-rs rejects an async cluster pipeline whose ops span slots. `Pipeline` groups ops by the slot of their first key, runs the groups concurrently and returns results in input order +- Pipeline ops take one key each. A multi-key op would span slots inside one group +- Command builders live in `commands.rs` and are shared by `RedisClient` and `Pipeline`, so a command is written once + +## Tests +- Against real Redis from `litellm-redis-testing`, never a mocked wire or asserted command bytes +- Every command test runs on each topology through `#[values(Topology::Standalone, Topology::Cluster)]`. A test is single-topology only when the behavior exists on one (a `db` number has no cluster meaning) +- Assert values and Redis state. Outages come from the container (`restart`, `freeze` / `thaw`), not `DEBUG`, which the official image disables +- Use a short client timeout in outage tests so they stay fast. Never sleep to wait for a state; poll with a deadline diff --git a/litellm-rust/crates/redis-client/Cargo.toml b/litellm-rust/crates/redis-client/Cargo.toml new file mode 100644 index 00000000000..eab57b36423 --- /dev/null +++ b/litellm-rust/crates/redis-client/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "litellm-redis-client" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-circuit-breaker.workspace = true +futures-util.workspace = true +redis = { workspace = true, features = ["cluster-async", "connection-manager", "tokio-rustls-comp"] } +thiserror.workspace = true + +[dev-dependencies] +litellm-redis-testing.workspace = true +rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/redis-client/src/client.rs b/litellm-rust/crates/redis-client/src/client.rs new file mode 100644 index 00000000000..a02b0847663 --- /dev/null +++ b/litellm-rust/crates/redis-client/src/client.rs @@ -0,0 +1,647 @@ +use std::{ + future::Future, + time::{Duration, Instant}, +}; + +use litellm_circuit_breaker::{Admission, Breaker, Failure}; +use redis::{Cmd, FromRedisValue, IntoConnectionInfo, ToRedisArgs, ToSingleRedisArg, Value}; + +use crate::{ + ConnectError, Endpoint, Error, Namespace, Pipeline, RedisConfig, Script, commands, + connection::Connection, +}; + +#[derive(Clone)] +pub struct RedisClient { + pub(crate) connection: Connection, + namespace: Namespace, + breaker: Option, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Ttl { + Missing, + Persistent, + Expires(Duration), +} + +impl RedisClient { + pub fn new(config: RedisConfig) -> Result { + let connection = match &config.endpoint { + Endpoint::Url(url) => { + Connection::standalone(url.as_str().into_connection_info()?, config.timeout)? + } + Endpoint::Host(host) => { + Connection::standalone(host.connection_info()?, config.timeout)? + } + Endpoint::Cluster(cluster) => Connection::cluster(cluster, config.timeout)?, + }; + Ok(Self { + connection, + namespace: config.namespace, + breaker: config.breaker, + }) + } + + pub async fn ping(&self) -> Result<(), Error> { + self.query(&commands::ping()).await + } + + pub async fn get(&self, key: &str) -> Result, Error> { + self.query(&commands::get(self.key(key))).await + } + + pub async fn mget(&self, keys: &[&str]) -> Result>, Error> { + if keys.is_empty() { + return Ok(Vec::new()); + } + self.query(&commands::mget(self.keys(keys))).await + } + + pub async fn set( + &self, + key: &str, + value: impl ToSingleRedisArg, + ttl: Duration, + ) -> Result<(), Error> { + self.query(&commands::set(self.key(key), value, ttl)).await + } + + pub async fn set_nx( + &self, + key: &str, + value: impl ToSingleRedisArg, + ttl: Duration, + ) -> Result { + let reply: Option = self + .query(&commands::set_nx(self.key(key), value, ttl)) + .await?; + Ok(reply.is_some()) + } + + pub async fn del(&self, keys: &[&str]) -> Result { + if keys.is_empty() { + return Ok(0); + } + self.query(&commands::del(self.keys(keys))).await + } + + pub async fn expire(&self, key: &str, ttl: Duration) -> Result { + self.query(&commands::expire(self.key(key), ttl)).await + } + + pub async fn ttl(&self, key: &str) -> Result { + let reply: i64 = self.query(&commands::ttl(self.key(key))).await?; + Ok(match reply { + -2 => Ttl::Missing, + -1 => Ttl::Persistent, + millis => Ttl::Expires(Duration::from_millis(millis.unsigned_abs())), + }) + } + + pub async fn incr_by(&self, key: &str, delta: i64) -> Result { + self.query(&commands::incr_by(self.key(key), delta)).await + } + + pub async fn incr_by_float(&self, key: &str, delta: f64) -> Result { + self.query(&commands::incr_by_float(self.key(key), delta)) + .await + } + + pub async fn rpush(&self, key: &str, values: impl ToRedisArgs) -> Result { + self.query(&commands::rpush(self.key(key), values)).await + } + + pub async fn eval( + &self, + script: &Script, + keys: &[&str], + args: impl ToRedisArgs, + ) -> Result { + let keys = self.keys(keys); + let args = args.to_redis_args(); + self.guarded(async { + Ok(script + .invocation(&keys, &args) + .invoke_async(&mut self.connection.clone()) + .await?) + }) + .await + } + + pub fn pipeline(&self) -> Pipeline { + Pipeline::new(self.namespace.clone()) + } + + pub async fn run(&self, pipeline: &Pipeline) -> Vec> { + let admission = match self.admit() { + Ok(admission) => admission, + Err(_) => { + return (0..pipeline.len()) + .map(|_| Err(Error::BreakerOpen)) + .collect(); + } + }; + let results = pipeline.run(&self.connection).await; + let transport = results.iter().find_map(|result| { + result + .as_ref() + .err() + .filter(|error| matches!(error, Error::Transport(_))) + }); + settle(admission, transport); + results + } + + fn key(&self, key: &str) -> String { + self.namespace.apply(key) + } + + fn keys(&self, keys: &[&str]) -> Vec { + keys.iter().map(|key| self.key(key)).collect() + } + + async fn query(&self, command: &Cmd) -> Result { + self.guarded(async { Ok(command.query_async(&mut self.connection.clone()).await?) }) + .await + } + + async fn guarded(&self, call: impl Future>) -> Result { + let admission = self.admit()?; + let result = call.await; + settle(admission, result.as_ref().err()); + result + } + + fn admit(&self) -> Result, Error> { + self.breaker + .as_ref() + .map(|breaker| { + breaker + .admit(Instant::now()) + .map_err(|_| Error::BreakerOpen) + }) + .transpose() + } +} + +fn settle(admission: Option, error: Option<&Error>) { + let Some(admission) = admission else { + return; + }; + match error { + None => admission.succeeded(), + Some(Error::Transport(error)) if error.is_timeout() => { + admission.failed(Failure::Timeout, Instant::now()) + } + Some(Error::Transport(_)) => admission.failed(Failure::Hard, Instant::now()), + Some(Error::Server(_) | Error::Decode(_) | Error::BreakerOpen) => drop(admission), + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use litellm_redis_testing::RedisServer; + use rstest::rstest; + + use litellm_circuit_breaker::{Breaker, Config, State}; + + use super::{RedisClient, Ttl}; + use crate::{ + DEFAULT_TIMEOUT, Endpoint, Error, HostEndpoint, Namespace, RedisConfig, + test_support::{Server, Topology, client_id}, + }; + + #[rstest] + #[tokio::test] + async fn namespaces_isolate_keys_on_the_server( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let a = server.client("a", DEFAULT_TIMEOUT).await; + let b = server.client("b", DEFAULT_TIMEOUT).await; + let bare = server.client("", DEFAULT_TIMEOUT).await; + + a.set("x", "1", Duration::from_secs(60)).await.unwrap(); + + assert_eq!(a.get::("x").await.unwrap().as_deref(), Some("1")); + assert_eq!(a.get::("a:x").await.unwrap().as_deref(), Some("1")); + assert_eq!( + bare.get::("a:x").await.unwrap().as_deref(), + Some("1") + ); + assert_eq!(b.get::("x").await.unwrap(), None); + } + + #[rstest] + #[tokio::test] + async fn a_rejected_command_keeps_the_connection( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", DEFAULT_TIMEOUT).await; + client.rpush("list", "item").await.unwrap(); + let before = client_id(&client, "list").await; + + let rejected = client.incr_by("list", 1).await; + + assert!(matches!(rejected, Err(Error::Server(_))), "{rejected:?}"); + assert_eq!(client.rpush("list", "next").await.unwrap(), 2); + assert_eq!(client_id(&client, "list").await, before); + } + + #[rstest] + #[tokio::test] + async fn a_reply_of_another_type_is_a_decode_error_and_a_missing_key_is_none( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", DEFAULT_TIMEOUT).await; + client + .set("float", "15.0", Duration::from_secs(60)) + .await + .unwrap(); + + let decoded = client.get::("float").await; + + assert!(matches!(decoded, Err(Error::Decode(_))), "{decoded:?}"); + assert_eq!(client.get::("missing").await.unwrap(), None); + } + + #[rstest] + #[tokio::test] + async fn a_timed_out_command_does_not_leak_its_reply_into_the_next_one( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", Duration::from_millis(500)).await; + client + .set("first", "1", Duration::from_secs(60)) + .await + .unwrap(); + client + .set("second", "2", Duration::from_secs(60)) + .await + .unwrap(); + + server.freeze().await; + let timed_out = client.get::("first").await; + server.thaw().await; + + assert!( + matches!(timed_out, Err(Error::Transport(_))), + "{timed_out:?}" + ); + assert_eq!( + client.get::("second").await.unwrap().as_deref(), + Some("2") + ); + } + + #[rstest] + #[tokio::test] + async fn the_same_client_recovers_after_redis_restarts( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", Duration::from_millis(500)).await; + + server.restart().await; + + let deadline = Instant::now() + Duration::from_secs(30); + let recovered = loop { + match client.incr_by("after_restart", 1).await { + Ok(value) => break Some(value), + Err(_) if Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(200)).await + } + Err(_) => break None, + } + }; + assert_eq!(recovered, Some(1)); + } + + #[rstest] + #[tokio::test] + async fn a_client_created_while_redis_is_down_works_once_redis_is_up( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + server.stop().await; + + let client = server.client("", Duration::from_millis(500)).await; + let while_down = client.ping().await; + server.resume().await; + + assert!( + matches!(while_down, Err(Error::Transport(_))), + "{while_down:?}" + ); + let deadline = Instant::now() + Duration::from_secs(30); + let recovered = loop { + match client.ping().await { + Ok(()) => break true, + Err(_) if Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(200)).await + } + Err(_) => break false, + } + }; + assert!(recovered); + } + + #[rstest] + #[tokio::test] + async fn empty_batches_succeed_without_asking_redis( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", DEFAULT_TIMEOUT).await; + + assert_eq!( + client.mget::(&[]).await.unwrap(), + Vec::>::new() + ); + assert_eq!(client.del(&[]).await.unwrap(), 0); + } + + #[rstest] + #[tokio::test] + async fn mget_and_del_span_slots_in_input_order( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", DEFAULT_TIMEOUT).await; + client.set("a", "1", Duration::from_secs(60)).await.unwrap(); + client.set("c", "3", Duration::from_secs(60)).await.unwrap(); + + let values = client.mget::(&["c", "b", "a"]).await.unwrap(); + let deleted = client.del(&["a", "b", "c"]).await.unwrap(); + + assert_eq!( + values, + vec![Some("3".to_owned()), None, Some("1".to_owned())] + ); + assert_eq!(deleted, 2); + assert_eq!( + client.mget::(&["a", "c"]).await.unwrap(), + vec![None, None] + ); + } + + #[rstest] + #[tokio::test] + async fn set_nx_only_writes_a_missing_key( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", DEFAULT_TIMEOUT).await; + + let first = client + .set_nx("lock", "owner-1", Duration::from_secs(60)) + .await + .unwrap(); + let second = client + .set_nx("lock", "owner-2", Duration::from_secs(60)) + .await + .unwrap(); + + assert!(first); + assert!(!second); + assert_eq!( + client.get::("lock").await.unwrap().as_deref(), + Some("owner-1") + ); + } + + #[rstest] + #[tokio::test] + async fn ttl_reports_missing_persistent_and_remaining_time( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", DEFAULT_TIMEOUT).await; + client.incr_by("persistent", 1).await.unwrap(); + client + .set("expiring", "1", Duration::from_secs(60)) + .await + .unwrap(); + client.incr_by_float("expired_later", 0.5).await.unwrap(); + assert!( + client + .expire("expired_later", Duration::from_secs(30)) + .await + .unwrap() + ); + + assert_eq!(client.ttl("missing").await.unwrap(), Ttl::Missing); + assert_eq!(client.ttl("persistent").await.unwrap(), Ttl::Persistent); + assert!(matches!( + client.ttl("expiring").await.unwrap(), + Ttl::Expires(left) if left > Duration::from_secs(50) && left <= Duration::from_secs(60) + )); + assert!(matches!( + client.ttl("expired_later").await.unwrap(), + Ttl::Expires(left) if left > Duration::from_secs(20) && left <= Duration::from_secs(30) + )); + assert!( + !client + .expire("missing", Duration::from_secs(30)) + .await + .unwrap() + ); + } + + #[rstest] + #[tokio::test] + async fn incr_by_float_accumulates_fractions( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", DEFAULT_TIMEOUT).await; + + client.incr_by_float("spend", 0.25).await.unwrap(); + + assert_eq!(client.incr_by_float("spend", 0.5).await.unwrap(), 0.75); + } + + #[rstest] + #[tokio::test] + async fn concurrent_commands_on_one_cloned_client_all_apply( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", DEFAULT_TIMEOUT).await; + + let tasks: Vec<_> = (0..100) + .map(|_| { + let client = client.clone(); + tokio::spawn(async move { + for _ in 0..10 { + client.incr_by("shared", 1).await.unwrap(); + } + }) + }) + .collect(); + for task in tasks { + task.await.unwrap(); + } + + assert_eq!(client.get::("shared").await.unwrap(), Some(1000)); + } + + fn breaker(failure_threshold: u32) -> Breaker { + Breaker::new( + Config { + failure_threshold, + recovery_timeout: Duration::from_millis(200), + timeout_min_duration: Duration::ZERO, + }, + Box::new(|_| {}), + ) + } + + #[rstest] + #[tokio::test] + async fn an_open_breaker_rejects_without_waiting_on_redis_and_a_probe_closes_it( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let breaker = breaker(2); + let client = server + .guarded_client("", Duration::from_millis(300), Some(breaker.clone())) + .await; + + server.freeze().await; + let first = client.get::("k").await; + let second = client.get::("k").await; + let started = Instant::now(); + let third = client.get::("k").await; + let rejected_in = started.elapsed(); + server.thaw().await; + + assert!(matches!(first, Err(Error::Transport(_))), "{first:?}"); + assert!(matches!(second, Err(Error::Transport(_))), "{second:?}"); + assert!(matches!(third, Err(Error::BreakerOpen)), "{third:?}"); + assert!(rejected_in < Duration::from_millis(50), "{rejected_in:?}"); + + let deadline = Instant::now() + Duration::from_secs(30); + let recovered = loop { + match client.get::("k").await { + Ok(value) => break Some(value), + Err(_) if Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(50)).await + } + Err(_) => break None, + } + }; + assert_eq!(recovered, Some(None)); + assert!( + matches!(breaker.state(), State::Closed { .. }), + "{:?}", + breaker.state() + ); + } + + #[rstest] + #[tokio::test] + async fn rejected_commands_never_open_the_breaker( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let breaker = breaker(2); + let client = server + .guarded_client("", DEFAULT_TIMEOUT, Some(breaker.clone())) + .await; + client.rpush("list", "item").await.unwrap(); + + let rejected = [ + client.incr_by("list", 1).await, + client.incr_by("list", 1).await, + client.incr_by("list", 1).await, + ]; + + assert!( + matches!(rejected[2], Err(Error::Server(_))), + "{:?}", + rejected[2] + ); + assert!( + matches!(breaker.state(), State::Closed { hard: 0, .. }), + "{:?}", + breaker.state() + ); + } + + #[rstest] + #[tokio::test] + async fn an_open_breaker_fails_every_pipeline_op_without_sending( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server + .guarded_client("", Duration::from_millis(300), Some(breaker(1))) + .await; + let pipeline = client.pipeline().incr_by("a", 1).incr_by("c", 1); + + server.freeze().await; + let failed = client.run(&pipeline).await; + let rejected = client.run(&pipeline).await; + server.thaw().await; + + assert!( + matches!(failed[0], Err(Error::Transport(_))), + "{:?}", + failed[0] + ); + assert!( + matches!(rejected[0], Err(Error::BreakerOpen)), + "{:?}", + rejected[0] + ); + assert!( + matches!(rejected[1], Err(Error::BreakerOpen)), + "{:?}", + rejected[1] + ); + } + + #[rstest] + #[tokio::test] + async fn a_host_endpoint_connects_to_its_database() { + let server = RedisServer::start().await.unwrap(); + let port = server.url().rsplit(':').next().unwrap().parse().unwrap(); + let connect = |endpoint| { + RedisClient::new(RedisConfig { + endpoint, + namespace: Namespace::default(), + timeout: DEFAULT_TIMEOUT, + breaker: None, + }) + }; + let in_db_3 = connect(Endpoint::Host(HostEndpoint { + host: "127.0.0.1".to_owned(), + port, + username: None, + password: None, + tls: false, + db: 3, + })) + .unwrap(); + let in_db_0 = connect(Endpoint::Url(server.url().to_owned())).unwrap(); + let via_url = connect(Endpoint::Url(format!("{}/3", server.url()))).unwrap(); + + in_db_3 + .set("k", "3", Duration::from_secs(60)) + .await + .unwrap(); + + assert_eq!(in_db_0.get::("k").await.unwrap(), None); + assert_eq!( + via_url.get::("k").await.unwrap().as_deref(), + Some("3") + ); + } +} diff --git a/litellm-rust/crates/redis-client/src/commands.rs b/litellm-rust/crates/redis-client/src/commands.rs new file mode 100644 index 00000000000..fb8a4a28c94 --- /dev/null +++ b/litellm-rust/crates/redis-client/src/commands.rs @@ -0,0 +1,62 @@ +use std::time::Duration; + +use redis::{Cmd, ToRedisArgs, ToSingleRedisArg}; + +pub(crate) fn ping() -> Cmd { + redis::cmd("PING") +} + +pub(crate) fn get(key: String) -> Cmd { + redis::cmd("GET").arg(key).to_owned() +} + +pub(crate) fn mget(keys: Vec) -> Cmd { + redis::cmd("MGET").arg(keys).to_owned() +} + +pub(crate) fn set(key: String, value: impl ToSingleRedisArg, ttl: Duration) -> Cmd { + redis::cmd("SET") + .arg(key) + .arg(value) + .arg("PX") + .arg(millis(ttl)) + .to_owned() +} + +pub(crate) fn set_nx(key: String, value: impl ToSingleRedisArg, ttl: Duration) -> Cmd { + redis::cmd("SET") + .arg(key) + .arg(value) + .arg("NX") + .arg("PX") + .arg(millis(ttl)) + .to_owned() +} + +pub(crate) fn del(keys: Vec) -> Cmd { + redis::cmd("DEL").arg(keys).to_owned() +} + +pub(crate) fn expire(key: String, ttl: Duration) -> Cmd { + redis::cmd("PEXPIRE").arg(key).arg(millis(ttl)).to_owned() +} + +pub(crate) fn ttl(key: String) -> Cmd { + redis::cmd("PTTL").arg(key).to_owned() +} + +pub(crate) fn incr_by(key: String, delta: i64) -> Cmd { + redis::cmd("INCRBY").arg(key).arg(delta).to_owned() +} + +pub(crate) fn incr_by_float(key: String, delta: f64) -> Cmd { + redis::cmd("INCRBYFLOAT").arg(key).arg(delta).to_owned() +} + +pub(crate) fn rpush(key: String, values: impl ToRedisArgs) -> Cmd { + redis::cmd("RPUSH").arg(key).arg(values).to_owned() +} + +fn millis(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} diff --git a/litellm-rust/crates/redis-client/src/config.rs b/litellm-rust/crates/redis-client/src/config.rs new file mode 100644 index 00000000000..64934c68891 --- /dev/null +++ b/litellm-rust/crates/redis-client/src/config.rs @@ -0,0 +1,152 @@ +use std::{fmt, time::Duration}; + +use redis::{ConnectionAddr, ConnectionInfo, IntoConnectionInfo, RedisConnectionInfo, RedisResult}; + +use litellm_circuit_breaker::Breaker; + +use crate::Namespace; + +pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5); + +pub struct RedisConfig { + pub endpoint: Endpoint, + pub namespace: Namespace, + pub timeout: Duration, + pub breaker: Option, +} + +#[derive(Clone, PartialEq)] +pub enum Endpoint { + Url(String), + Host(HostEndpoint), + Cluster(ClusterEndpoint), +} + +#[derive(Clone, PartialEq)] +pub struct HostEndpoint { + pub host: String, + pub port: u16, + pub username: Option, + pub password: Option, + pub tls: bool, + pub db: i64, +} + +#[derive(Clone, PartialEq)] +pub struct ClusterEndpoint { + pub nodes: Vec, + pub username: Option, + pub password: Option, + pub tls: bool, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ClusterNode { + pub host: String, + pub port: u16, +} + +impl ClusterEndpoint { + pub(crate) fn addrs(&self) -> impl Iterator + '_ { + self.nodes + .iter() + .map(|node| addr(&node.host, node.port, self.tls)) + } +} + +impl HostEndpoint { + pub(crate) fn connection_info(&self) -> RedisResult { + let settings = RedisConnectionInfo::default().set_db(self.db); + let settings = match &self.username { + Some(username) => settings.set_username(username), + None => settings, + }; + let settings = match &self.password { + Some(password) => settings.set_password(password), + None => settings, + }; + Ok(addr(&self.host, self.port, self.tls) + .into_connection_info()? + .set_redis_settings(settings)) + } +} + +fn addr(host: &str, port: u16, tls: bool) -> ConnectionAddr { + match tls { + true => ConnectionAddr::TcpTls { + host: host.to_owned(), + port, + insecure: false, + tls_params: None, + }, + false => ConnectionAddr::Tcp(host.to_owned(), port), + } +} + +impl fmt::Debug for Endpoint { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Url(url) => formatter + .debug_tuple("Url") + .field(&redacted_url(url)) + .finish(), + Self::Host(host) => formatter + .debug_struct("Host") + .field("host", &host.host) + .field("port", &host.port) + .field("username", &host.username) + .field("password", &host.password.as_ref().map(|_| "")) + .field("tls", &host.tls) + .field("db", &host.db) + .finish(), + Self::Cluster(cluster) => formatter + .debug_struct("Cluster") + .field("nodes", &cluster.nodes) + .field("username", &cluster.username) + .field("password", &cluster.password.as_ref().map(|_| "")) + .field("tls", &cluster.tls) + .finish(), + } + } +} + +fn redacted_url(url: &str) -> String { + match url + .split_once("://") + .and_then(|(scheme, rest)| Some((scheme, rest.rsplit_once('@')?.1))) + { + Some((scheme, host)) => format!("{scheme}://@{host}"), + None => url.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::{Endpoint, HostEndpoint}; + + #[rstest] + #[case::url_with_password(Endpoint::Url("redis://user:secret@cache:6379/0".to_owned()), "redis://@cache:6379/0")] + #[case::url_with_at_in_password(Endpoint::Url("rediss://:p@ss@cache:6380".to_owned()), "rediss://@cache:6380")] + #[case::host_password( + Endpoint::Host(HostEndpoint { + host: "cache".to_owned(), + port: 6379, + username: None, + password: Some("secret".to_owned()), + tls: false, + db: 0, + }), + "" + )] + fn debug_output_never_shows_the_password(#[case] endpoint: Endpoint, #[case] shown: &str) { + let debug = format!("{endpoint:?}"); + + assert!( + !debug.contains("secret") && !debug.contains("p@ss"), + "{debug}" + ); + assert!(debug.contains(shown), "{debug}"); + } +} diff --git a/litellm-rust/crates/redis-client/src/connection.rs b/litellm-rust/crates/redis-client/src/connection.rs new file mode 100644 index 00000000000..aaf26f105da --- /dev/null +++ b/litellm-rust/crates/redis-client/src/connection.rs @@ -0,0 +1,80 @@ +use std::time::Duration; + +use redis::{ + Cmd, ConnectionInfo, Pipeline, RedisFuture, RedisResult, Value, + aio::{ConnectionLike, ConnectionManager, ConnectionManagerConfig}, + cluster::{ClusterClientBuilder, ClusterConfig}, + cluster_async::ClusterConnection, +}; + +use crate::ClusterEndpoint; + +const RETRIES: u32 = 3; + +#[derive(Clone)] +pub(crate) enum Connection { + Standalone(ConnectionManager), + Cluster(ClusterConnection), +} + +impl Connection { + pub(crate) fn standalone(info: ConnectionInfo, timeout: Duration) -> RedisResult { + let config = ConnectionManagerConfig::new() + .set_connection_timeout(Some(timeout)) + .set_response_timeout(Some(timeout)) + .set_number_of_retries(RETRIES as usize); + let manager = ConnectionManager::new_lazy_with_config(redis::Client::open(info)?, config)?; + Ok(Self::Standalone(manager)) + } + + pub(crate) fn cluster(endpoint: &ClusterEndpoint, timeout: Duration) -> RedisResult { + let builder = ClusterClientBuilder::new(endpoint.addrs()) + .connection_timeout(timeout) + .response_timeout(timeout) + .retries(RETRIES); + let builder = match &endpoint.username { + Some(username) => builder.username(username), + None => builder, + }; + let builder = match &endpoint.password { + Some(password) => builder.password(password), + None => builder, + }; + let config = ClusterConfig::new() + .set_connection_timeout(timeout) + .set_response_timeout(timeout); + Ok(Self::Cluster( + builder + .build()? + .get_pending_async_connection_with_config(config), + )) + } +} + +impl ConnectionLike for Connection { + fn req_packed_command<'a>(&'a mut self, cmd: &'a Cmd) -> RedisFuture<'a, Value> { + match self { + Self::Standalone(connection) => connection.req_packed_command(cmd), + Self::Cluster(connection) => connection.req_packed_command(cmd), + } + } + + fn req_packed_commands<'a>( + &'a mut self, + pipeline: &'a Pipeline, + offset: usize, + count: usize, + ) -> RedisFuture<'a, Vec> { + match self { + Self::Standalone(connection) => connection.req_packed_commands(pipeline, offset, count), + Self::Cluster(connection) => connection.req_packed_commands(pipeline, offset, count), + } + } + + fn get_db(&self) -> i64 { + match self { + Self::Standalone(connection) => connection.get_db(), + Self::Cluster(connection) => connection.get_db(), + } + } +} diff --git a/litellm-rust/crates/redis-client/src/error.rs b/litellm-rust/crates/redis-client/src/error.rs new file mode 100644 index 00000000000..ed3b2c74004 --- /dev/null +++ b/litellm-rust/crates/redis-client/src/error.rs @@ -0,0 +1,69 @@ +use redis::{ErrorKind, RedisError, ServerErrorKind}; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Redis did not answer, so the command may or may not have applied")] + Transport(#[source] RedisError), + #[error("Redis rejected the command")] + Server(#[source] RedisError), + #[error("Redis replied with a value of a different type")] + Decode(#[source] RedisError), + #[error("the Redis circuit breaker is open, so the command was not sent")] + BreakerOpen, +} + +#[derive(Debug, thiserror::Error)] +#[error("connecting to Redis")] +pub struct ConnectError(#[from] RedisError); + +impl From for Error { + fn from(error: RedisError) -> Self { + match error.kind() { + ErrorKind::Server( + ServerErrorKind::BusyLoading + | ServerErrorKind::ClusterDown + | ServerErrorKind::MasterDown + | ServerErrorKind::TryAgain + | ServerErrorKind::ReadOnly, + ) => Self::Transport(error), + ErrorKind::Server(_) | ErrorKind::Extension => Self::Server(error), + ErrorKind::Parse | ErrorKind::UnexpectedReturnType => Self::Decode(error), + _ => Self::Transport(error), + } + } +} + +#[cfg(test)] +mod tests { + use redis::{RedisError, Value}; + use rstest::rstest; + + use super::Error; + + fn reply(resp: &str) -> RedisError { + match redis::parse_redis_value(resp.as_bytes()).unwrap() { + Value::ServerError(error) => error.into(), + other => panic!("{resp:?} parsed as {other:?}"), + } + } + + #[rstest] + #[case::wrong_type("-WRONGTYPE Operation against a key holding the wrong kind of value\r\n")] + #[case::no_script("-NOSCRIPT No matching script\r\n")] + #[case::cross_slot("-CROSSSLOT Keys in request don't hash to the same slot\r\n")] + #[case::generic("-ERR value is not an integer or out of range\r\n")] + #[case::no_perm("-NOPERM this user has no permissions to access one of the keys\r\n")] + fn rejections_are_server_errors(#[case] resp: &str) { + assert!(matches!(Error::from(reply(resp)), Error::Server(_))); + } + + #[rstest] + #[case::loading("-LOADING Redis is loading the dataset in memory\r\n")] + #[case::cluster_down("-CLUSTERDOWN The cluster is down\r\n")] + #[case::master_down("-MASTERDOWN Link with MASTER is down\r\n")] + #[case::try_again("-TRYAGAIN Multiple keys request during rehashing of slot\r\n")] + #[case::read_only("-READONLY You can't write against a read only replica.\r\n")] + fn unavailable_nodes_are_transport_errors(#[case] resp: &str) { + assert!(matches!(Error::from(reply(resp)), Error::Transport(_))); + } +} diff --git a/litellm-rust/crates/redis-client/src/lib.rs b/litellm-rust/crates/redis-client/src/lib.rs new file mode 100644 index 00000000000..47ff8677e06 --- /dev/null +++ b/litellm-rust/crates/redis-client/src/lib.rs @@ -0,0 +1,20 @@ +mod client; +mod commands; +mod config; +mod connection; +mod error; +mod namespace; +mod pipeline; +mod script; +#[cfg(test)] +mod test_support; + +pub use client::{RedisClient, Ttl}; +pub use config::{ + ClusterEndpoint, ClusterNode, DEFAULT_TIMEOUT, Endpoint, HostEndpoint, RedisConfig, +}; +pub use error::{ConnectError, Error}; +pub use namespace::Namespace; +pub use pipeline::Pipeline; +pub use redis::Value; +pub use script::Script; diff --git a/litellm-rust/crates/redis-client/src/namespace.rs b/litellm-rust/crates/redis-client/src/namespace.rs new file mode 100644 index 00000000000..4db02b5b7bb --- /dev/null +++ b/litellm-rust/crates/redis-client/src/namespace.rs @@ -0,0 +1,57 @@ +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Namespace(Option); + +impl Namespace { + pub fn new(namespace: impl Into) -> Self { + let namespace = namespace.into(); + Self((!namespace.is_empty()).then_some(namespace)) + } + + pub fn as_str(&self) -> Option<&str> { + self.0.as_deref() + } + + pub(crate) fn apply(&self, key: &str) -> String { + match &self.0 { + Some(namespace) if !is_prefixed(key, namespace) => { + format!("{namespace}:{key}") + } + _ => key.to_owned(), + } + } +} + +fn is_prefixed(key: &str, namespace: &str) -> bool { + key.strip_prefix(namespace) + .is_some_and(|rest| rest.starts_with(':')) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::Namespace; + + #[rstest] + #[case::plain(Namespace::new("litellm"), "spend:key:abc", "litellm:spend:key:abc")] + #[case::no_namespace(Namespace::default(), "spend:key:abc", "spend:key:abc")] + #[case::empty_namespace(Namespace::new(""), "spend:key:abc", "spend:key:abc")] + #[case::already_prefixed(Namespace::new("litellm"), "litellm:abc", "litellm:abc")] + #[case::shares_prefix_without_delimiter( + Namespace::new("litellm"), + "litellm_spend_update_buffer", + "litellm:litellm_spend_update_buffer" + )] + #[case::hash_tagged_key( + Namespace::new("litellm"), + "{api_key:sk-1}:requests", + "litellm:{api_key:sk-1}:requests" + )] + fn prefixes_keys_like_python_check_and_fix_namespace( + #[case] namespace: Namespace, + #[case] key: &str, + #[case] expected: &str, + ) { + assert_eq!(namespace.apply(key), expected); + } +} diff --git a/litellm-rust/crates/redis-client/src/pipeline.rs b/litellm-rust/crates/redis-client/src/pipeline.rs new file mode 100644 index 00000000000..e54c65e9605 --- /dev/null +++ b/litellm-rust/crates/redis-client/src/pipeline.rs @@ -0,0 +1,292 @@ +use std::{collections::HashMap, time::Duration}; + +use futures_util::future::join_all; +use redis::{ + Cmd, ErrorKind, RedisResult, ServerErrorKind, ToRedisArgs, ToSingleRedisArg, Value, + cluster_routing::Slot, +}; + +use crate::{Error, Namespace, Script, commands, connection::Connection}; + +pub struct Pipeline { + namespace: Namespace, + ops: Vec, +} + +struct Op { + command: Cmd, + key: String, + script: Option, +} + +struct ScriptCall { + script: Script, + keys: Vec, + args: Vec>, +} + +type Settled = (usize, Result); + +impl Pipeline { + pub(crate) fn new(namespace: Namespace) -> Self { + Self { + namespace, + ops: Vec::new(), + } + } + + pub fn get(self, key: &str) -> Self { + let key = self.namespace.apply(key); + self.command(commands::get(key.clone()), key) + } + + pub fn set(self, key: &str, value: impl ToSingleRedisArg, ttl: Duration) -> Self { + let key = self.namespace.apply(key); + self.command(commands::set(key.clone(), value, ttl), key) + } + + pub fn del(self, key: &str) -> Self { + let key = self.namespace.apply(key); + self.command(commands::del(vec![key.clone()]), key) + } + + pub fn expire(self, key: &str, ttl: Duration) -> Self { + let key = self.namespace.apply(key); + self.command(commands::expire(key.clone(), ttl), key) + } + + pub fn incr_by(self, key: &str, delta: i64) -> Self { + let key = self.namespace.apply(key); + self.command(commands::incr_by(key.clone(), delta), key) + } + + pub fn incr_by_float(self, key: &str, delta: f64) -> Self { + let key = self.namespace.apply(key); + self.command(commands::incr_by_float(key.clone(), delta), key) + } + + pub fn eval(self, script: &Script, keys: &[&str], args: impl ToRedisArgs) -> Self { + let keys: Vec = keys.iter().map(|key| self.namespace.apply(key)).collect(); + let args = args.to_redis_args(); + let op = Op { + command: script.evalsha(&keys, &args), + key: keys.first().cloned().unwrap_or_default(), + script: Some(ScriptCall { + script: script.clone(), + keys, + args, + }), + }; + self.push(op) + } + + fn command(self, command: Cmd, key: String) -> Self { + self.push(Op { + command, + key, + script: None, + }) + } + + fn push(mut self, op: Op) -> Self { + self.ops.push(op); + self + } + + pub(crate) fn len(&self) -> usize { + self.ops.len() + } + + pub(crate) async fn run(&self, connection: &Connection) -> Vec> { + let groups = self.groups(connection); + let settled = join_all( + groups + .iter() + .map(|group| self.run_group(connection.clone(), group)), + ) + .await; + let mut settled: Vec = settled.into_iter().flatten().collect(); + settled.sort_by_key(|(index, _)| *index); + settled.into_iter().map(|(_, result)| result).collect() + } + + fn groups(&self, connection: &Connection) -> Vec> { + match connection { + _ if self.ops.is_empty() => Vec::new(), + Connection::Standalone(_) => vec![(0..self.ops.len()).collect()], + Connection::Cluster(_) => { + let mut by_slot: HashMap> = HashMap::new(); + for (index, op) in self.ops.iter().enumerate() { + by_slot + .entry(Slot::for_key(&op.key)) + .or_default() + .push(index); + } + by_slot.into_values().collect() + } + } + } + + async fn run_group(&self, mut connection: Connection, indices: &[usize]) -> Vec { + let mut pipeline = redis::pipe(); + pipeline.ignore_errors(); + for &index in indices { + pipeline.add_command(self.ops[index].command.clone()); + } + let replies: RedisResult>> = + pipeline.query_async(&mut connection).await; + let replies = match replies { + Ok(replies) => replies, + Err(error) => { + return indices + .iter() + .map(|&index| (index, Err(Error::from(error.clone())))) + .collect(); + } + }; + let mut settled = Vec::with_capacity(indices.len()); + for (&index, reply) in indices.iter().zip(replies) { + settled.push((index, self.settle(&mut connection, index, reply).await)); + } + settled + } + + async fn settle( + &self, + connection: &mut Connection, + index: usize, + reply: RedisResult, + ) -> Result { + match (reply, &self.ops[index].script) { + (Err(error), Some(call)) + if error.kind() == ErrorKind::Server(ServerErrorKind::NoScript) => + { + Ok(call + .script + .invocation(&call.keys, &call.args) + .invoke_async(connection) + .await?) + } + (reply, _) => Ok(reply?), + } + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use redis::Value; + use rstest::rstest; + + use crate::{ + DEFAULT_TIMEOUT, Error, Script, + test_support::{Server, Topology, flush_scripts}, + }; + + #[rstest] + #[tokio::test] + async fn each_op_settles_on_its_own_in_input_order( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", DEFAULT_TIMEOUT).await; + client + .set("text", "not a number", Duration::from_secs(60)) + .await + .unwrap(); + let pipeline = client + .pipeline() + .incr_by("a", 1) + .incr_by("text", 1) + .incr_by("c", 5) + .set("b", "2", Duration::from_secs(60)) + .get("b") + .incr_by("a", 1) + .get("missing"); + + let results = client.run(&pipeline).await; + + assert_eq!(results.len(), 7); + assert_eq!(results[0].as_ref().unwrap(), &Value::Int(1)); + assert!( + matches!(results[1], Err(Error::Server(_))), + "{:?}", + results[1] + ); + assert_eq!(results[2].as_ref().unwrap(), &Value::Int(5)); + assert_eq!(results[3].as_ref().unwrap(), &Value::Okay); + assert_eq!( + results[4].as_ref().unwrap(), + &Value::BulkString(b"2".to_vec()) + ); + assert_eq!(results[5].as_ref().unwrap(), &Value::Int(2)); + assert_eq!(results[6].as_ref().unwrap(), &Value::Nil); + } + + #[rstest] + #[tokio::test] + async fn a_flushed_script_in_a_pipeline_reruns_alone( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("ns", DEFAULT_TIMEOUT).await; + let incr = Script::new("return redis.call('INCRBY', KEYS[1], ARGV[1])"); + client.eval::(&incr, &["counter"], 1).await.unwrap(); + flush_scripts(&client).await; + let pipeline = client + .pipeline() + .eval(&incr, &["counter"], 2) + .incr_by("other", 7); + + let results = client.run(&pipeline).await; + + assert_eq!(results[0].as_ref().unwrap(), &Value::Int(3)); + assert_eq!(results[1].as_ref().unwrap(), &Value::Int(7)); + assert_eq!(client.get::("counter").await.unwrap(), Some(3)); + } + + #[rstest] + #[tokio::test] + async fn an_unreachable_server_fails_every_op_as_transport( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", Duration::from_millis(500)).await; + let pipeline = client.pipeline().incr_by("a", 1).incr_by("c", 1); + + server.freeze().await; + let results = client.run(&pipeline).await; + server.thaw().await; + + assert_eq!(results.len(), 2); + assert!( + matches!(results[0], Err(Error::Transport(_))), + "{:?}", + results[0] + ); + assert!( + matches!(results[1], Err(Error::Transport(_))), + "{:?}", + results[1] + ); + } + + #[rstest] + #[tokio::test] + async fn an_empty_pipeline_returns_without_asking_redis( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", Duration::from_secs(2)).await; + + server.freeze().await; + let started = Instant::now(); + let results = client.run(&client.pipeline()).await; + let elapsed = started.elapsed(); + server.thaw().await; + + assert!(results.is_empty()); + assert!(elapsed < Duration::from_secs(1), "{elapsed:?}"); + } +} diff --git a/litellm-rust/crates/redis-client/src/script.rs b/litellm-rust/crates/redis-client/src/script.rs new file mode 100644 index 00000000000..c5d73133ecc --- /dev/null +++ b/litellm-rust/crates/redis-client/src/script.rs @@ -0,0 +1,88 @@ +use redis::{Cmd, ScriptInvocation}; + +#[derive(Clone)] +pub struct Script(redis::Script); + +impl Script { + pub fn new(source: &str) -> Self { + Self(redis::Script::new(source)) + } + + pub(crate) fn evalsha(&self, keys: &[String], args: &[Vec]) -> Cmd { + redis::cmd("EVALSHA") + .arg(self.0.get_hash()) + .arg(keys.len()) + .arg(keys) + .arg(args) + .to_owned() + } + + pub(crate) fn invocation(&self, keys: &[String], args: &[Vec]) -> ScriptInvocation<'_> { + let mut invocation = self.0.prepare_invoke(); + invocation.key(keys).arg(args); + invocation + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use rstest::rstest; + + use super::Script; + use crate::{ + DEFAULT_TIMEOUT, Error, + test_support::{Server, Topology, flush_scripts}, + }; + + #[rstest] + #[tokio::test] + async fn script_keys_are_namespaced( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("a", DEFAULT_TIMEOUT).await; + let bare = server.client("", DEFAULT_TIMEOUT).await; + let read = Script::new("return redis.call('GET', KEYS[1])"); + bare.set("a:x", "1", Duration::from_secs(60)).await.unwrap(); + + let value: Option = client + .eval(&read, &["x"], Vec::::new()) + .await + .unwrap(); + + assert_eq!(value.as_deref(), Some("1")); + } + + #[rstest] + #[tokio::test] + async fn a_failing_script_is_a_server_error_and_keeps_the_connection( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", DEFAULT_TIMEOUT).await; + let incr = Script::new("return redis.call('INCRBY', KEYS[1], ARGV[1])"); + client.rpush("list", "item").await.unwrap(); + + let failed = client.eval::(&incr, &["list"], 1).await; + + assert!(matches!(failed, Err(Error::Server(_))), "{failed:?}"); + assert_eq!(client.eval::(&incr, &["counter"], 3).await.unwrap(), 3); + } + + #[rstest] + #[tokio::test] + async fn a_flushed_script_is_loaded_again( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server.client("", DEFAULT_TIMEOUT).await; + let incr = Script::new("return redis.call('INCRBY', KEYS[1], ARGV[1])"); + client.eval::(&incr, &["counter"], 1).await.unwrap(); + + flush_scripts(&client).await; + + assert_eq!(client.eval::(&incr, &["counter"], 1).await.unwrap(), 2); + } +} diff --git a/litellm-rust/crates/redis-client/src/test_support.rs b/litellm-rust/crates/redis-client/src/test_support.rs new file mode 100644 index 00000000000..64a174b1543 --- /dev/null +++ b/litellm-rust/crates/redis-client/src/test_support.rs @@ -0,0 +1,125 @@ +use std::time::Duration; + +use litellm_circuit_breaker::Breaker; +use litellm_redis_testing::{RedisCluster, RedisServer}; +use redis::cluster_routing::{Route, RoutingInfo, SingleNodeRoutingInfo, SlotAddr}; + +use crate::{ + ClusterEndpoint, ClusterNode, Endpoint, Namespace, RedisClient, RedisConfig, + connection::Connection, +}; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum Topology { + Standalone, + Cluster, +} + +pub(crate) enum Server { + Standalone(RedisServer), + Cluster(RedisCluster), +} + +impl Server { + pub(crate) async fn start(topology: Topology) -> Self { + match topology { + Topology::Standalone => Self::Standalone(RedisServer::start().await.unwrap()), + Topology::Cluster => Self::Cluster(RedisCluster::start().await.unwrap()), + } + } + + pub(crate) fn endpoint(&self) -> Endpoint { + match self { + Self::Standalone(server) => Endpoint::Url(server.url().to_owned()), + Self::Cluster(cluster) => Endpoint::Cluster(ClusterEndpoint { + nodes: cluster + .nodes() + .map(|(host, port)| ClusterNode { + host: host.to_owned(), + port, + }) + .collect(), + username: None, + password: None, + tls: false, + }), + } + } + + pub(crate) async fn client(&self, namespace: &str, timeout: Duration) -> RedisClient { + self.guarded_client(namespace, timeout, None).await + } + + pub(crate) async fn guarded_client( + &self, + namespace: &str, + timeout: Duration, + breaker: Option, + ) -> RedisClient { + RedisClient::new(RedisConfig { + endpoint: self.endpoint(), + namespace: Namespace::new(namespace), + timeout, + breaker, + }) + .unwrap() + } + + pub(crate) async fn restart(&self) { + match self { + Self::Standalone(server) => server.restart().await.unwrap(), + Self::Cluster(cluster) => cluster.restart().await.unwrap(), + } + } + + pub(crate) async fn stop(&self) { + match self { + Self::Standalone(server) => server.stop().await.unwrap(), + Self::Cluster(cluster) => cluster.stop().await.unwrap(), + } + } + + pub(crate) async fn resume(&self) { + match self { + Self::Standalone(server) => server.resume().await.unwrap(), + Self::Cluster(cluster) => cluster.resume().await.unwrap(), + } + } + + pub(crate) async fn freeze(&self) { + match self { + Self::Standalone(server) => server.freeze().await.unwrap(), + Self::Cluster(cluster) => cluster.freeze().await.unwrap(), + } + } + + pub(crate) async fn thaw(&self) { + match self { + Self::Standalone(server) => server.thaw().await.unwrap(), + Self::Cluster(cluster) => cluster.thaw().await.unwrap(), + } + } +} + +pub(crate) async fn client_id(client: &RedisClient, key: &str) -> i64 { + let command = redis::cmd("CLIENT").arg("ID").to_owned(); + match client.connection.clone() { + Connection::Standalone(mut connection) => { + command.query_async(&mut connection).await.unwrap() + } + Connection::Cluster(mut connection) => { + let node = Route::with_key(key, SlotAddr::Master); + let routing = RoutingInfo::SingleNode(SingleNodeRoutingInfo::SpecificNode(node)); + redis::from_redis_value(connection.route_command(command, routing).await.unwrap()) + .unwrap() + } + } +} + +pub(crate) async fn flush_scripts(client: &RedisClient) { + redis::cmd("SCRIPT") + .arg("FLUSH") + .query_async::<()>(&mut client.connection.clone()) + .await + .unwrap(); +} diff --git a/litellm-rust/crates/redis-testing/AGENTS.md b/litellm-rust/crates/redis-testing/AGENTS.md new file mode 100644 index 00000000000..fb7a4cd737a --- /dev/null +++ b/litellm-rust/crates/redis-testing/AGENTS.md @@ -0,0 +1,6 @@ +- Dev-only crate that starts real Redis for tests. Never a dependency outside `[dev-dependencies]` +- Same shape as `db-testing`: one container per test, a handle that removes it on drop +- The host port is fixed at start so `restart` keeps the same URL and a client can reconnect to it +- `RedisCluster` runs 3 masters in one container. Each node listens on the same port inside and outside and announces 127.0.0.1, so MOVED redirects work from the test process +- Outages come from the container (`restart`, `freeze` / `thaw`), never `DEBUG`, which the official image disables +- No env gate that makes a test pass without Redis. A test that can't start its container fails diff --git a/litellm-rust/crates/redis-testing/Cargo.toml b/litellm-rust/crates/redis-testing/Cargo.toml new file mode 100644 index 00000000000..c9bad227acc --- /dev/null +++ b/litellm-rust/crates/redis-testing/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "litellm-redis-testing" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +testcontainers-modules = { workspace = true, features = ["redis"] } +thiserror.workspace = true +tokio = { workspace = true, features = ["io-util"] } diff --git a/litellm-rust/crates/redis-testing/src/cluster.rs b/litellm-rust/crates/redis-testing/src/cluster.rs new file mode 100644 index 00000000000..e61f7832b1c --- /dev/null +++ b/litellm-rust/crates/redis-testing/src/cluster.rs @@ -0,0 +1,93 @@ +use testcontainers_modules::testcontainers::{ + ContainerAsync, GenericImage, ImageExt, + core::{IntoContainerPort, WaitFor}, + runners::AsyncRunner, +}; + +use crate::{ + Error, + ports::{HOST, start_on_free_ports, wait_for_pong}, + standalone::REDIS_TAG, +}; + +const READY: &str = "redis-testing cluster ready"; + +const START_CLUSTER: &str = r#" +set -e +bus=17000 +for port in "$@"; do + mkdir -p "/data/$port" + redis-server --port "$port" --cluster-enabled yes --cluster-port "$bus" \ + --cluster-config-file "/data/$port/nodes.conf" --cluster-announce-ip 127.0.0.1 \ + --dir "/data/$port" --save '' --appendonly no --daemonize yes + bus=$((bus + 1)) +done +for port in "$@"; do + until redis-cli -p "$port" ping >/dev/null 2>&1; do sleep 0.1; done +done +if [ ! -f /data/created ]; then + redis-cli --cluster create $(for port in "$@"; do printf '127.0.0.1:%s ' "$port"; done) --cluster-yes >/dev/null + touch /data/created +fi +for port in "$@"; do + until redis-cli -p "$port" cluster info | grep -q cluster_state:ok; do sleep 0.1; done +done +echo "redis-testing cluster ready" +exec sleep infinity +"#; + +pub struct RedisCluster { + container: ContainerAsync, + ports: [u16; 3], +} + +impl RedisCluster { + pub async fn start() -> Result { + start_on_free_ports(Self::start_on).await + } + + async fn start_on(ports: [u16; 3]) -> Result { + let image = GenericImage::new("redis", REDIS_TAG) + .with_wait_for(WaitFor::message_on_stdout(READY)) + .with_entrypoint("sh"); + let request = ports.iter().fold( + image.with_cmd( + ["-c", START_CLUSTER, "sh"] + .into_iter() + .map(str::to_owned) + .chain(ports.iter().map(u16::to_string)), + ), + |request, &port| request.with_mapped_port(port, port.tcp()), + ); + let container = request.start().await?; + for port in ports { + wait_for_pong(port).await?; + } + Ok(Self { container, ports }) + } + + pub fn nodes(&self) -> impl Iterator + '_ { + self.ports.iter().map(|&port| (HOST, port)) + } + + pub async fn restart(&self) -> Result<(), Error> { + self.stop().await?; + self.resume().await + } + + pub async fn stop(&self) -> Result<(), Error> { + Ok(self.container.stop().await?) + } + + pub async fn resume(&self) -> Result<(), Error> { + Ok(self.container.start().await?) + } + + pub async fn freeze(&self) -> Result<(), Error> { + Ok(self.container.pause().await?) + } + + pub async fn thaw(&self) -> Result<(), Error> { + Ok(self.container.unpause().await?) + } +} diff --git a/litellm-rust/crates/redis-testing/src/error.rs b/litellm-rust/crates/redis-testing/src/error.rs new file mode 100644 index 00000000000..f21cc959812 --- /dev/null +++ b/litellm-rust/crates/redis-testing/src/error.rs @@ -0,0 +1,11 @@ +use std::io; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("reserving a host port for the Redis container")] + Port(#[source] io::Error), + #[error("Redis on host port {port} did not answer PING")] + Unreachable { port: u16 }, + #[error("controlling the Redis container")] + Container(#[from] testcontainers_modules::testcontainers::TestcontainersError), +} diff --git a/litellm-rust/crates/redis-testing/src/lib.rs b/litellm-rust/crates/redis-testing/src/lib.rs new file mode 100644 index 00000000000..6c2afd7f275 --- /dev/null +++ b/litellm-rust/crates/redis-testing/src/lib.rs @@ -0,0 +1,8 @@ +mod cluster; +mod error; +mod ports; +mod standalone; + +pub use cluster::RedisCluster; +pub use error::Error; +pub use standalone::RedisServer; diff --git a/litellm-rust/crates/redis-testing/src/ports.rs b/litellm-rust/crates/redis-testing/src/ports.rs new file mode 100644 index 00000000000..e7b94988c10 --- /dev/null +++ b/litellm-rust/crates/redis-testing/src/ports.rs @@ -0,0 +1,66 @@ +use std::{ + future::Future, + io, + net::TcpListener, + time::{Duration, Instant}, +}; + +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, +}; + +use crate::Error; + +pub(crate) const HOST: &str = "127.0.0.1"; +const READY_TIMEOUT: Duration = Duration::from_secs(10); +const START_ATTEMPTS: usize = 3; + +fn free_ports() -> Result<[u16; N], Error> { + let listeners = (0..N) + .map(|_| TcpListener::bind((HOST, 0))) + .collect::>>() + .map_err(Error::Port)?; + let ports = listeners + .iter() + .map(|listener| listener.local_addr().map(|address| address.port())) + .collect::>>() + .map_err(Error::Port)?; + Ok(ports + .try_into() + .unwrap_or_else(|ports: Vec| unreachable!("{} ports", ports.len()))) +} + +pub(crate) async fn wait_for_pong(port: u16) -> Result<(), Error> { + let deadline = Instant::now() + READY_TIMEOUT; + while Instant::now() < deadline { + if ping(port).await.unwrap_or(false) { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(Error::Unreachable { port }) +} + +async fn ping(port: u16) -> io::Result { + let mut stream = TcpStream::connect((HOST, port)).await?; + stream.write_all(b"PING\r\n").await?; + let mut reply = [0; 7]; + stream.read_exact(&mut reply).await?; + Ok(&reply == b"+PONG\r\n") +} + +pub(crate) async fn start_on_free_ports( + start: impl Fn([u16; N]) -> F, +) -> Result +where + F: Future>, +{ + let mut attempt = 1; + loop { + match start(free_ports()?).await { + Err(_) if attempt < START_ATTEMPTS => attempt += 1, + started => return started, + } + } +} diff --git a/litellm-rust/crates/redis-testing/src/standalone.rs b/litellm-rust/crates/redis-testing/src/standalone.rs new file mode 100644 index 00000000000..8060ccbf190 --- /dev/null +++ b/litellm-rust/crates/redis-testing/src/standalone.rs @@ -0,0 +1,62 @@ +use testcontainers_modules::{ + redis::{REDIS_PORT, Redis}, + testcontainers::core::IntoContainerPort, + testcontainers::{ContainerAsync, ImageExt, runners::AsyncRunner}, +}; + +use crate::{ + Error, + ports::{HOST, start_on_free_ports, wait_for_pong}, +}; + +pub(crate) const REDIS_TAG: &str = + "7.4@sha256:44960b3100c48159a66d28c18b0588c193492e25f7551d60e51dd464fc3eca5b"; + +pub struct RedisServer { + container: ContainerAsync, + url: String, +} + +impl RedisServer { + pub async fn start() -> Result { + start_on_free_ports(Self::start_on).await + } + + async fn start_on([host_port]: [u16; 1]) -> Result { + let container = Redis::default() + .with_tag(REDIS_TAG) + .with_mapped_port(host_port, REDIS_PORT.tcp()) + .start() + .await?; + wait_for_pong(host_port).await?; + Ok(Self { + container, + url: format!("redis://{HOST}:{host_port}"), + }) + } + + pub fn url(&self) -> &str { + &self.url + } + + pub async fn restart(&self) -> Result<(), Error> { + self.stop().await?; + self.resume().await + } + + pub async fn stop(&self) -> Result<(), Error> { + Ok(self.container.stop().await?) + } + + pub async fn resume(&self) -> Result<(), Error> { + Ok(self.container.start().await?) + } + + pub async fn freeze(&self) -> Result<(), Error> { + Ok(self.container.pause().await?) + } + + pub async fn thaw(&self) -> Result<(), Error> { + Ok(self.container.unpause().await?) + } +} diff --git a/litellm-rust/crates/spend-clickhouse/Cargo.toml b/litellm-rust/crates/spend-clickhouse/Cargo.toml index 661a75ac02e..893c5daded3 100644 --- a/litellm-rust/crates/spend-clickhouse/Cargo.toml +++ b/litellm-rust/crates/spend-clickhouse/Cargo.toml @@ -19,7 +19,7 @@ time = { workspace = true, features = ["formatting"] } [dev-dependencies] litellm-http = { workspace = true, features = ["test-support"] } rstest.workspace = true -testcontainers-modules = { version = "0.15.0", features = ["clickhouse"] } +testcontainers-modules = { workspace = true, features = ["clickhouse"] } tokio.workspace = true uuid.workspace = true wiremock.workspace = true diff --git a/litellm-rust/crates/storage-clickhouse/Cargo.toml b/litellm-rust/crates/storage-clickhouse/Cargo.toml index 4e72e823dc2..39d04f2de5d 100644 --- a/litellm-rust/crates/storage-clickhouse/Cargo.toml +++ b/litellm-rust/crates/storage-clickhouse/Cargo.toml @@ -19,6 +19,6 @@ url.workspace = true [dev-dependencies] litellm-http = { workspace = true, features = ["test-support"] } rstest.workspace = true -testcontainers-modules = { version = "0.15.0", features = ["clickhouse"] } +testcontainers-modules = { workspace = true, features = ["clickhouse"] } tokio.workspace = true wiremock.workspace = true diff --git a/litellm/__init__.py b/litellm/__init__.py index c0141f43611..117ba9604e9 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1785,7 +1785,7 @@ if TYPE_CHECKING: from .llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig as AnthropicMessagesConfig, ) - from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + from .llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig, ) from .llms.bedrock.messages.mantle_transformation import ( @@ -1808,7 +1808,7 @@ if TYPE_CHECKING: from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( VertexAIAnthropicConfig as VertexAIAnthropicConfig, ) - from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import ( + from .llms.vertex_ai.vertex_ai_partner_models.llama.transformation import ( VertexAILlama3Config as VertexAILlama3Config, ) from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import ( @@ -1835,10 +1835,10 @@ if TYPE_CHECKING: from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import ( AmazonQwen3Config as AmazonQwen3Config, ) - from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import ( + from .llms.bedrock.chat.invoke_transformations.anthropic_legacy_text_completions_transformation import ( AmazonAnthropicConfig as AmazonAnthropicConfig, ) - from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + from .llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig, ) from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import ( @@ -2000,7 +2000,7 @@ if TYPE_CHECKING: GradientAIConfig as GradientAIConfig, ) from .llms.openai.chat.gpt_transformation import OpenAIGPTConfig as OpenAIGPTConfig - from .llms.openai.chat.gpt_5_transformation import ( + from .llms.openai.chat.gpt_reasoning_transformation import ( OpenAIGPT5Config as OpenAIGPT5Config, ) from .llms.openai.transcriptions.whisper_transformation import ( @@ -2141,7 +2141,7 @@ if TYPE_CHECKING: from .llms.azure.chat.gpt_transformation import ( AzureOpenAIConfig as AzureOpenAIConfig, ) - from .llms.azure.chat.gpt_5_transformation import ( + from .llms.azure.chat.gpt_reasoning_transformation import ( AzureOpenAIGPT5Config as AzureOpenAIGPT5Config, ) from .llms.azure.completion.transformation import ( diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index e0cb7c680bc..a56004d1274 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -805,7 +805,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "BedrockClaudePlatformMessagesConfig", ), "AmazonAnthropicClaudeMessagesConfig": ( - ".llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation", + ".llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation", "AmazonAnthropicClaudeMessagesConfig", ), "AmazonMantleMessagesConfig": ( @@ -835,7 +835,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "VertexAIAnthropicConfig", ), "VertexAILlama3Config": ( - ".llms.vertex_ai.vertex_ai_partner_models.llama3.transformation", + ".llms.vertex_ai.vertex_ai_partner_models.llama.transformation", "VertexAILlama3Config", ), "VertexAIAi21Config": ( @@ -880,11 +880,11 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "GoogleAIStudioGeminiConfig", ), # Alias "AmazonAnthropicConfig": ( - ".llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation", + ".llms.bedrock.chat.invoke_transformations.anthropic_legacy_text_completions_transformation", "AmazonAnthropicConfig", ), "AmazonAnthropicClaudeConfig": ( - ".llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation", + ".llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation", "AmazonAnthropicClaudeConfig", ), "AmazonCohereConfig": ( @@ -1101,7 +1101,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "OpenAIOSeriesConfig", ), # Alias "OpenAIGPTConfig": (".llms.openai.chat.gpt_transformation", "OpenAIGPTConfig"), - "OpenAIGPT5Config": (".llms.openai.chat.gpt_5_transformation", "OpenAIGPT5Config"), + "OpenAIGPT5Config": (".llms.openai.chat.gpt_reasoning_transformation", "OpenAIGPT5Config"), "OpenAIWhisperAudioTranscriptionConfig": ( ".llms.openai.transcriptions.whisper_transformation", "OpenAIWhisperAudioTranscriptionConfig", @@ -1175,7 +1175,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "CometAPIConfig": (".llms.cometapi.chat.transformation", "CometAPIConfig"), "AzureOpenAIConfig": (".llms.azure.chat.gpt_transformation", "AzureOpenAIConfig"), "AzureOpenAIGPT5Config": ( - ".llms.azure.chat.gpt_5_transformation", + ".llms.azure.chat.gpt_reasoning_transformation", "AzureOpenAIGPT5Config", ), "AzureOpenAITextConfig": ( diff --git a/litellm/_uuid.py b/litellm/_uuid.py index e9578b7287f..55df233b553 100644 --- a/litellm/_uuid.py +++ b/litellm/_uuid.py @@ -1,10 +1,14 @@ """ Internal unified UUID helper. -Always uses fastuuid for performance. +Uses fastuuid where supported and the standard library on Python 3.15+. """ -import fastuuid as _uuid +import sys +import uuid as _uuid + +if sys.version_info < (3, 15): + import fastuuid as _uuid # Expose a module-like alias so callers can use: uuid.uuid4() uuid = _uuid diff --git a/litellm/constants.py b/litellm/constants.py index d1daf2ee2d1..2901b7d8948 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -627,7 +627,7 @@ BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BU # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) -DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 4096) +DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096 TOGETHER_AI_4_B: Final = get_env_int("TOGETHER_AI_4_B", 4) TOGETHER_AI_8_B: Final = get_env_int("TOGETHER_AI_8_B", 8) diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index abbeccc10b3..3b73f65c917 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -456,10 +456,14 @@ def prepare_call( extra_headers: Mapping[str, str] | None, kwargs: Mapping[str, object], ) -> DecisionsCall: + base_url_alias: Final = kwargs.get("base_url") + call_api_base: Final = ( + base_url_alias if api_base is None and isinstance(base_url_alias, str) and base_url_alias else api_base + ) upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, - api_base=api_base, + api_base=call_api_base, api_key=api_key, ) provider_config: Final = _provider_config(upstream_model, provider) @@ -488,7 +492,7 @@ def prepare_call( try: resolved_api_base, static_api_key = provider_config.resolve_connection( - dynamic_api_base or api_base, dynamic_api_key or api_key + dynamic_api_base or call_api_base, dynamic_api_key or api_key ) except ValueError as error: raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error diff --git a/litellm/integrations/otel/model/destination.py b/litellm/integrations/otel/model/destination.py index 4226b4361dd..e760ba3600b 100644 --- a/litellm/integrations/otel/model/destination.py +++ b/litellm/integrations/otel/model/destination.py @@ -42,7 +42,7 @@ class OtelDestination(LiteLLMBaseModel): success_sampling_rate: float | None = Field( default=None, description=( - "Share of the request trees forwarded, 0.0..1.0, drawn once per request when its root span ends; " + "Share of the request trees forwarded, 0.0..1.0, picked by a hash of the trace id; " "``None`` forwards every one." ), ) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 898eab5715b..fc1e9b7099d 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -1,13 +1,14 @@ """Provider / exporter factory + the Baggage span processor.""" import queue -import random import threading import time +import weakref from collections import OrderedDict from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from hashlib import sha256 from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal +from typing import TYPE_CHECKING, Final, Literal, TypeAlias from opentelemetry import _logs, baggage, metrics, trace from opentelemetry._logs import Logger, LoggerProvider, NoOpLoggerProvider @@ -486,15 +487,15 @@ def is_llm_call_span(span: ReadableSpan) -> bool: return GenAI.OPERATION_NAME in attributes and MCP.METHOD_NAME not in attributes -#: Request trees held back for a sampled destination until the last span of the trace +#: Request trees held back for a sampled destination until the last span of the request #: still open ends, the verdicts kept for the spans that start after that (a post-call -#: database write), and the spans in flight per trace. All bounded, so a span that never -#: ends, or a flood of requests, cannot hold spans for ever: the oldest tree is decided -#: on what it has. +#: database write), the spans in flight per request, and the request each open span +#: belongs to. All bounded, so a span that never ends, or a flood of requests, cannot +#: hold spans for ever: the oldest tree is decided on what it has. _MAX_PENDING_TREES: Final = 1024 _MAX_PENDING_SPANS_PER_TREE: Final = 512 _MAX_REMEMBERED_VERDICTS: Final = 4096 -_MAX_OPEN_TRACES: Final = 16384 +_MAX_OPEN_REQUESTS: Final = 16384 class _PendingTree: @@ -512,25 +513,35 @@ def _is_sampled(destination: "OtelDestination") -> bool: return destination.success_sampling_rate is not None or destination.error_sampling_rate is not None -#: A verdict is one request tree's draw for one destination at its rates: the exporter +#: A request is its trace id and the id of its root, the span of it whose parent is +#: remote or missing. Calls sharing a caller's trace are separate requests, each decided +#: on its own outcome. +_RequestKey: TypeAlias = tuple[int, int] + +#: A verdict is one request's keep or drop for one destination at its rates: the exporter #: ``cache_key`` leaves the rates out, so two destinations sharing an exporter with -#: different rates would otherwise share one draw. -_VerdictKey = tuple[int, object, float | None, float | None] +#: different rates would otherwise share one verdict. +_VerdictKey: TypeAlias = tuple[int, int, object, float | None, float | None] -def _verdict_key(trace_id: int, destination: "OtelDestination") -> _VerdictKey: - return (trace_id, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate) +def _verdict_key(request: _RequestKey, destination: "OtelDestination") -> _VerdictKey: + return (*request, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate) -def _keeps_tree(destination: "OtelDestination", failed: bool, draw: Callable[[], float]) -> bool: - """Whether one draw against the destination's rate keeps a request tree, read the - way the legacy Arize callback reads ``arize_success_sampling_rate`` / - ``arize_error_sampling_rate``: a tree with a failed span answers to the error - rate, an unset rate keeps everything and ``0.0`` keeps nothing.""" +def _keeps_tree(destination: "OtelDestination", failed: bool, trace_id: int) -> bool: + """Whether the destination's rate keeps a request tree of ``trace_id``: a tree with + a failed span answers to ``arize_error_sampling_rate``, the rest to + ``arize_success_sampling_rate``, an unset rate keeps everything and ``0.0`` keeps + nothing. + + The first eight bytes of a SHA-256 of the trace id are read against the rate, so + every process deciding a request of the trace with the same outcome reaches the same + verdict.""" rate: Final = destination.error_sampling_rate if failed else destination.success_sampling_rate if rate is None: return True - return rate > 0.0 and draw() <= rate + digest: Final = sha256(trace_id.to_bytes(16, "big")).digest() + return int.from_bytes(digest[:8], "big") < round(rate * 2**64) def _in_scope(span: ReadableSpan, scope: "OtelSpanScope") -> bool: @@ -667,10 +678,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): default_capture_by_backend: Mapping[str, str] = MappingProxyType({}), pending_drains: int = _MAX_PENDING_DRAINS, drain_pool: _DrainPool | None = None, - sampling_draw: Callable[[], float] | None = None, ) -> None: self._operator_sinks: Final = operator_sinks - self._draw: Final = sampling_draw if sampling_draw is not None else random.random self._excluded_db_systems: Final = excluded_db_systems self._default_capture: Final = default_capture self._default_capture_by_backend: Final = default_capture_by_backend @@ -682,24 +691,32 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count - self._pending: OrderedDict[int, _PendingTree] = OrderedDict() # mutable-ok: bounded, by trace id - self._open: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, spans in flight by trace id + self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request + self._open: OrderedDict[_RequestKey, set[int]] = OrderedDict() # mutable-ok: bounded, open span ids by request + self._roots: dict[int, _RequestKey] = {} # mutable-ok: holds only the spans ``_open`` holds, request by span id + self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary() self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains) def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: context: Final = span.context - trace_id: Final = context.trace_id if context is not None else 0 with self._lock: - self._open[trace_id] = self._open.get(trace_id, 0) + 1 - while len(self._open) > _MAX_OPEN_TRACES: - self._open.popitem(last=False) + request: Final = self._request_of(span, trace.get_current_span(parent_context)) + if context is not None: + self._span_roots[span] = request[1] + self._roots[context.span_id] = request + self._open.setdefault(request, set()).add(context.span_id) + while len(self._open) > _MAX_OPEN_REQUESTS: + for forgotten in self._open.popitem(last=False)[1]: + self._roots.pop(forgotten, None) def on_end(self, span: ReadableSpan) -> None: suppressed: Final = suppressed_backends() attributes: Final = span.attributes or _NO_ATTRIBUTES - context: Final = span.context - trace_id: Final = context.trace_id if context is not None else 0 + with self._lock: + request: Final = self._request_of(span) + if span.context is not None: + self._roots.pop(span.context.span_id, None) failed: Final = span.status.status_code is StatusCode.ERROR for destination in request_destinations(): if ( @@ -716,11 +733,32 @@ class TenantFanOutSpanProcessor(SpanProcessor): if not _is_sampled(destination): self._forward(span, destination) continue - for held_span, held_destination in self._route(trace_id, span, destination, failed): + for held_span, held_destination in self._route(request, span, destination, failed): self._forward(held_span, held_destination) - for held_span, held_destination in self._settle(trace_id, failed): + for held_span, held_destination in self._settle(request, span, failed): self._forward(held_span, held_destination) + def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey: + """The request ``span`` belongs to, read under ``_lock``: the request recorded while + it is open, else the span itself when its parent is remote or missing, else its + parent's request, recorded while the parent is open or on ``parent_span`` for a + child starting under a parent that has ended, else the parent itself.""" + context: Final = span.context + if context is None: + return (0, 0) + recorded: Final = self._roots.get(context.span_id) + if recorded is not None: + return recorded + parent: Final = span.parent + if parent is None or parent.is_remote: + return (context.trace_id, context.span_id) + parent_request: Final = self._roots.get(parent.span_id) + if parent_request is not None: + return parent_request + if isinstance(parent_span, SDKSpan): + return (context.trace_id, self._span_roots.get(parent_span, parent.span_id)) + return (context.trace_id, parent.span_id) + def _default_capture_for(self, destination: "OtelDestination") -> str: backend: Final = destination.callback_name return ( @@ -743,14 +781,14 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._release(processor) def _route( - self, trace_id: int, span: ReadableSpan, destination: "OtelDestination", failed: bool + self, request: _RequestKey, span: ReadableSpan, destination: "OtelDestination", failed: bool ) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": """The spans to forward now for ``span`` on a sampled ``destination``: the span itself when its tree's verdict is already known and keeps it, nothing while the - tree is held for the rest of its trace, or what the bounds made the fan-out decide + tree is held for the rest of its request, or what the bounds made the fan-out decide early (a tree that outgrew its cap is decided on what it has, and so is the oldest tree once too many are waiting, so a span that never ends holds nothing back for - ever). A failed span marks its tree before any such early draw. Once shutdown + ever). A failed span marks its tree before any such early decision. Once shutdown has started flushing, nothing is held: a span ending then is decided at once, since no later flush would reach it. @@ -758,58 +796,62 @@ class TenantFanOutSpanProcessor(SpanProcessor): another thread between the two would leave this span in a fresh tree. """ with self._lock: - verdict: Final = self._verdicts.get(_verdict_key(trace_id, destination)) + verdict: Final = self._verdicts.get(_verdict_key(request, destination)) if verdict is not None: return ((span, destination),) if verdict else () - tree: Final = self._pending.setdefault(trace_id, _PendingTree()) + tree: Final = self._pending.setdefault(request, _PendingTree()) tree.held.append((span, destination)) if failed: tree.failed = True if self._draining or len(tree.held) >= _MAX_PENDING_SPANS_PER_TREE: - return self._decide_locked(trace_id) + return self._decide_locked(request) if len(self._pending) > _MAX_PENDING_TREES: oldest: Final = next(iter(self._pending)) return self._decide_locked(oldest) return () - def _decide(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + def _decide(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": with self._lock: - return self._decide_locked(trace_id) + return self._decide_locked(request) - def _settle(self, trace_id: int, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": - """What the trace's held tree exports once the span that just ended was the last - of its trace still open. The model-call span ends in the post-call callback, - after the server span, so a tree is decided when its trace goes quiet rather than + def _settle( + self, request: _RequestKey, span: ReadableSpan, failed: bool + ) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + """What the request's held tree exports once the span that just ended was the last + of its request still open. The model-call span ends in the post-call callback, + after the server span, so a tree is decided when its request goes quiet rather than when its root ends, and a model call that fails late still answers to the error - rate. A trace whose starts this fan-out never saw (attached mid-flight, or forgotten - by the bound) is decided as each of its spans ends. + rate. A request whose starts this fan-out never saw (attached mid-flight, or + forgotten by the bound) is decided as each of its spans ends. """ with self._lock: - tree: Final = self._pending.get(trace_id) + tree: Final = self._pending.get(request) if tree is not None and failed: tree.failed = True - still_open: Final = self._open.pop(trace_id, 1) - 1 - if still_open > 0: - self._open[trace_id] = still_open + still_open: Final = self._open.pop(request, set()) + if span.context is not None: + still_open.discard(span.context.span_id) + if still_open: + self._open[request] = still_open return () - return self._decide_locked(trace_id) + return self._decide_locked(request) - def _decide_locked(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": - """Draw once per destination for the held tree; the spans the draws keep, and a - verdict remembered for the spans of this trace that start later.""" - tree: Final = self._pending.pop(trace_id, None) + def _decide_locked(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + """Decide once per destination for the held tree; the spans the verdicts keep, and + a verdict remembered for the spans of this request that start later.""" + tree: Final = self._pending.pop(request, None) if tree is None: return () - verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one draw per destination of this tree + verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one verdict per destination of this tree for _, destination in tree.held: - key = _verdict_key(trace_id, destination) + key = _verdict_key(request, destination) if key not in verdicts: - verdicts[key] = _keeps_tree(destination, tree.failed, self._draw) + verdicts[key] = _keeps_tree(destination, tree.failed, request[0]) self._verdicts[key] = verdicts[key] while len(self._verdicts) > _MAX_REMEMBERED_VERDICTS: self._verdicts.popitem(last=False) return tuple( - (span, destination) for span, destination in tree.held if verdicts[_verdict_key(trace_id, destination)] + (span, destination) for span, destination in tree.held if verdicts[_verdict_key(request, destination)] ) def _operator_already_writes( @@ -850,8 +892,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): with self._lock: self._draining = True undecided: Final = tuple(self._pending) - for trace_id in undecided: - for held_span, held_destination in self._decide(trace_id): + for request in undecided: + for held_span, held_destination in self._decide(request): self._forward(held_span, held_destination) deadline: Final = time.monotonic() + self._drain_seconds with self._lock: diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 19d161b8aa4..bc8344fa686 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -14,6 +14,7 @@ from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast +import openai from pydantic import BaseModel, Field, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict, assert_never @@ -22,11 +23,13 @@ from litellm._internal_context import with_service_target from litellm._logging import print_verbose, verbose_logger from litellm.constants import ( AUTH_OBJECTS_TARGET, + LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, PROMETHEUS_OVERFLOW_SERIES_LABEL_VALUE, PROXY_LLM_PROVIDER_FALLBACK, PROXY_REJECTED_BEFORE_ROUTING_KEY, ) from litellm.exceptions import ( + CallerCredentialAuthenticationError, MalformedRequestFieldError, validate_rate_limit_category, validate_rate_limit_type, @@ -51,6 +54,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_cust from litellm.litellm_core_utils.service_tier_utils import ( get_service_tier_from_standard_logging_payload, ) +from litellm.llms.github_copilot.per_user_auth import github_copilot_user_session_from from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.models.end_user import LiteLLM_EndUserTable from litellm.models.team import LiteLLM_TeamTable @@ -269,7 +273,10 @@ _NO_LOGGED_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) def _logged_fields(value: object) -> Mapping[str, object]: - return _LOGGED_FIELDS.validate_python(value) if isinstance(value, Mapping) else _NO_LOGGED_FIELDS + try: + return _LOGGED_FIELDS.validate_python(value) + except ValidationError: + return _NO_LOGGED_FIELDS def _is_llm_provider(name: str) -> bool: @@ -359,6 +366,87 @@ def _get_proxy_llm_router() -> Router | None: return llm_router +_CALLER_CREDENTIAL_FIELDS: Final = frozenset( + { + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_web_identity_token", + "azure_ad_token", + "azure_password", + "azure_scope", + "azure_username", + "client_id", + "client_secret", + "gigachat_access_token", + "litellm_credential_name", + "oci_fingerprint", + "oci_key", + "oci_key_file", + "oci_tenancy", + "oci_user", + "tenant_id", + "token", + "user_config", + "vertex_ai_credentials", + "vertex_credentials", + "watsonx_credentials", + "wx_credentials", + "zen_api_key", + } +) + + +_CALLER_ENDPOINT_FIELDS: Final = frozenset( + {"aws_bedrock_runtime_endpoint", "aws_sts_endpoint", "deployment_url", "s3_endpoint_url", "sagemaker_base_url"} +) + + +_CALLER_TRANSPORT_FIELDS: Final = frozenset( + {"api_key", "extra_headers", "headers", "organization", "provider_specific_header"} +) + + +def _sent_a_provider_credential_or_endpoint(caller_body: Mapping[str, object]) -> bool: + """True when the caller's request body, or its ``extra_body``, carries a provider credential field or a + provider endpoint field.""" + sent: Final = {*caller_body, *_logged_fields(caller_body.get("extra_body"))} + return not (_CALLER_CREDENTIAL_FIELDS.isdisjoint(sent) and _CALLER_ENDPOINT_FIELDS.isdisjoint(sent)) + + +def _used_a_caller_credential(logged_call: Mapping[str, object]) -> bool: + """True when the provider was called with a credential or at an endpoint that belongs to the caller rather than + the deployment: an api_key, header, OpenAI organization or other provider credential or endpoint field the caller + sent with the request, a stored credential the caller picked by name, the caller's own Anthropic OAuth token, the caller's own GitHub Copilot connection, or the + caller's identity token that a token-exchange deployment trades in on every call.""" + litellm_params: Final = _logged_fields(logged_call.get("litellm_params")) + proxy_server_request: Final = _logged_fields(litellm_params.get("proxy_server_request")) + caller_credential_fields: Final = proxy_server_request.get("caller_credential_fields") + logged_metadata: Final = _logged_fields(_logged_fields(logged_call.get("standard_logging_object")).get("metadata")) + return ( + ( + isinstance(caller_credential_fields, tuple) + and any(field in caller_credential_fields for field in _CALLER_TRANSPORT_FIELDS) + ) + or _sent_a_provider_credential_or_endpoint(_logged_fields(proxy_server_request.get("body"))) + or bool(litellm_params.get("token_exchange_endpoint")) + or logged_metadata.get("used_client_oauth_token") is True + or github_copilot_user_session_from(litellm_params) is not None + ) + + +def _is_deployment_failure(exception: openai.APIError, logged_call: Mapping[str, object]) -> bool: + """True for an error raised while calling the deployment with its own credential, on a request the proxy + did not reject itself. An error LiteLLM raises before it calls the provider, such as a missing deployment key, + is not one.""" + return not ( + isinstance(exception, CallerCredentialAuthenticationError) + or bool(logged_call.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)) + or logged_call.get("api_call_start_time") is None + or _used_a_caller_credential(logged_call) + ) + + def _bounded_requested_model_label(requested_model: object, router_originated: bool = False) -> str | None: """ Bound ``requested_model`` label cardinality: names the router recognizes @@ -2817,6 +2905,8 @@ class PrometheusLogger(CustomLogger): information from various callback function signatures and determines if the request represents an invalid API key attempt that should be filtered from metrics. + An error raised while calling the deployment with its own credential is never skipped. + Args: kwargs: Dictionary potentially containing exception and other data user_api_key_dict: User API key authentication object (currently unused) @@ -2836,6 +2926,11 @@ class PrometheusLogger(CustomLogger): if exception is None and kwargs: exception = kwargs.get("exception") + if isinstance(exception, openai.APIError) and _is_deployment_failure( + exception, _LOGGED_FIELDS.validate_python(kwargs or {}) + ): + return False + if self._is_invalid_api_key_request(status_code, exception=exception): verbose_logger.debug( "Skipping Prometheus metrics for invalid API key request: status_code=%s, exception=%s", diff --git a/litellm/litellm_core_utils/duration_parser.py b/litellm/litellm_core_utils/duration_parser.py index d52716951df..95bc77e3eba 100644 --- a/litellm/litellm_core_utils/duration_parser.py +++ b/litellm/litellm_core_utils/duration_parser.py @@ -22,7 +22,7 @@ _BUDGET_DURATION_WORD_ALIASES: Final[dict[str, str]] = { } -def _normalize_duration(duration: str) -> str: +def normalize_duration(duration: str) -> str: return _BUDGET_DURATION_WORD_ALIASES.get(duration.strip().lower(), duration) @@ -61,7 +61,7 @@ def duration_in_seconds(duration: str) -> int: Returns time in seconds till when budget needs to be reset """ - value, unit = _extract_from_regex(duration=_normalize_duration(duration)) + value, unit = _extract_from_regex(duration=normalize_duration(duration)) if unit == "s": return value @@ -89,7 +89,7 @@ def duration_in_seconds_from(start: datetime, duration: str) -> int: month of a different length needs its own start to know when it ends. Every other unit is a fixed length and ignores `start`. """ - value, unit = _extract_from_regex(duration=_normalize_duration(duration)) + value, unit = _extract_from_regex(duration=normalize_duration(duration)) if unit == "mo": return _months_in_seconds(start=start, months=value) return duration_in_seconds(duration) @@ -129,7 +129,7 @@ def get_next_standardized_reset_time( current_time, _ = _setup_timezone(current_time, timezone_str) # Parse duration - value, unit = _parse_duration(_normalize_duration(duration)) + value, unit = _parse_duration(normalize_duration(duration)) if value is None: verbose_logger.warning( "Unrecognized budget_duration %r; falling back to a next-midnight reset. " @@ -171,7 +171,7 @@ def get_budget_window_start(duration: str, reset_at: datetime) -> datetime: """Start of the budget period that ends at `reset_at`, under the same rules `get_next_standardized_reset_time` used to pick it: `30d` and `Nmo` span calendar months and a duration it does not recognize resets at the next midnight.""" - value, unit = _parse_duration(_normalize_duration(duration)) + value, unit = _parse_duration(normalize_duration(duration)) if value is None: return reset_at - timedelta(days=1) match unit: diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 85557090ae9..ca8e2af18e6 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -76,6 +76,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "itpm", "otpm", "use_xai_oauth", + "chatgpt_auth_file", "github_copilot_auth_type", "github_copilot_host", "github_copilot_user_session", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index de51ecd0b21..19c1d5c414c 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -158,6 +158,14 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None +def resolve_model_and_provider_for_metadata(model: str, custom_llm_provider: str | None = None) -> tuple[str, str]: + declared: Final = declared_authenticating_provider(model, custom_llm_provider) + if declared is not None: + return model.removeprefix(f"{declared}/"), declared + resolved_model, resolved_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + return resolved_model, resolved_provider + + def inferred_provider(model: str | None) -> str | None: if not model: return None @@ -842,7 +850,9 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, custom_llm_provider, - ) = litellm.ChatGPTConfig().get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider) + ) = litellm.ChatGPTConfig().get_openai_compatible_provider_info( + model, api_base, api_key, custom_llm_provider, litellm_params + ) elif custom_llm_provider == "novita": api_base = api_base or get_secret("NOVITA_API_BASE") or "https://api.novita.ai/v3/openai" dynamic_api_key = api_key or get_secret_str("NOVITA_API_KEY") diff --git a/litellm/litellm_core_utils/get_provider_specific_headers.py b/litellm/litellm_core_utils/get_provider_specific_headers.py index bc325c3c702..62dfcb073e5 100644 --- a/litellm/litellm_core_utils/get_provider_specific_headers.py +++ b/litellm/litellm_core_utils/get_provider_specific_headers.py @@ -1,10 +1,23 @@ from collections.abc import Mapping, Sequence from typing import Final +from pydantic import TypeAdapter + from litellm.types.utils import ProviderSpecificHeader +_PROVIDER_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]] = TypeAdapter( + ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None +) + class ProviderSpecificHeaderUtils: + @staticmethod + def forwarded_authorization(provider_specific_header: object, custom_llm_provider: str | None) -> str | None: + headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( + _PROVIDER_HEADERS.validate_python(provider_specific_header), custom_llm_provider + ) + return next((value for key, value in headers.items() if key.lower() == "authorization"), None) + @staticmethod def get_provider_specific_headers( provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None, diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 0aed2956b0a..3bc8dc4e23e 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1490,6 +1490,7 @@ def convert_to_gemini_tool_call_result( message: ChatCompletionToolMessage | ChatCompletionFunctionMessage, last_message_with_tool_calls: dict | None, forward_function_call_id: bool = False, + supports_multimodal_function_responses: bool = True, ) -> VertexPartType | list[VertexPartType]: """ OpenAI message with a tool result looks like: @@ -1652,9 +1653,9 @@ def convert_to_gemini_tool_call_result( _part: Final[VertexPartType] = {"function_response": _function_response} - # For multimodal function responses, Gemini expects media parts nested - # inside functionResponse.parts instead of sibling content parts. if inline_data_list: + if not supports_multimodal_function_responses: + return [_part, *(VertexPartType(inline_data=inline_data) for inline_data in inline_data_list)] _function_response["parts"] = [{"inline_data": inline_data} for inline_data in inline_data_list] return [_part] diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_reasoning_transformation.py similarity index 98% rename from litellm/llms/azure/chat/gpt_5_transformation.py rename to litellm/llms/azure/chat/gpt_reasoning_transformation.py index 4bec335064e..5bd03f511c5 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_reasoning_transformation.py @@ -4,7 +4,7 @@ from collections.abc import Collection from typing import ClassVar, Final import litellm -from litellm.llms.openai.chat.gpt_5_transformation import ( +from litellm.llms.openai.chat.gpt_reasoning_transformation import ( OpenAIGPT5Config, is_gpt_reasoning_series_name, ) diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 03fe27437a5..26ae30635d4 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -16,7 +16,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_azure_openai_messages, ) from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.llms.openai.chat.gpt_5_transformation import GPT_REASONING_SERIES_MARKERS +from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS from litellm.types.llms.azure import ( API_VERSION_MONTH_SUPPORTED_RESPONSE_FORMAT, API_VERSION_YEAR_SUPPORTED_RESPONSE_FORMAT, diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 6d904503df8..647ad97e02c 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -12,7 +12,7 @@ from pydantic import TypeAdapter from litellm._logging import verbose_logger from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION from litellm.litellm_core_utils.url_utils import encode_url_path_segment -from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config +from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import * diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 3b541310d28..6aed7eb3c54 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -21,7 +21,7 @@ from litellm.llms.azure_ai.common_utils import ( is_foundry_model_inference_base, ) from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj -from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig from litellm.llms.xai.chat.transformation import XAIChatConfig diff --git a/litellm/llms/azure_ai/common_utils.py b/litellm/llms/azure_ai/common_utils.py index cffe9049de6..65d73363b30 100644 --- a/litellm/llms/azure_ai/common_utils.py +++ b/litellm/llms/azure_ai/common_utils.py @@ -6,7 +6,7 @@ from urllib.parse import urlparse import litellm from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter -from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py similarity index 100% rename from litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py rename to litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_legacy_text_completions_transformation.py similarity index 100% rename from litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude2_transformation.py rename to litellm/llms/bedrock/chat/invoke_transformations/anthropic_legacy_text_completions_transformation.py diff --git a/litellm/llms/bedrock/chat/mantle/transformation.py b/litellm/llms/bedrock/chat/mantle/transformation.py index a1aa2034e4c..b5a352bab48 100644 --- a/litellm/llms/bedrock/chat/mantle/transformation.py +++ b/litellm/llms/bedrock/chat/mantle/transformation.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Any, Final from litellm.litellm_core_utils.prompt_templates.image_handling import async_inline_remote_media from litellm.llms.anthropic.chat.transformation import JsonModeRequest -from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( +from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) from litellm.llms.bedrock.common_utils import build_mantle_messages_url diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index cb10c2384cd..8c4702bcf99 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -1052,7 +1052,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): # --- Anthropic: use existing AmazonAnthropicClaudeConfig --- if provider == LlmProviders.ANTHROPIC: - from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py similarity index 100% rename from litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py rename to litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py diff --git a/litellm/llms/bedrock/messages/mantle_transformation.py b/litellm/llms/bedrock/messages/mantle_transformation.py index d4a2d2eabe9..7b9dde42c4a 100644 --- a/litellm/llms/bedrock/messages/mantle_transformation.py +++ b/litellm/llms/bedrock/messages/mantle_transformation.py @@ -19,7 +19,7 @@ from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.bedrock.common_utils import build_mantle_messages_url -from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( +from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( diff --git a/litellm/llms/bedrock_mantle/chat/transformation.py b/litellm/llms/bedrock_mantle/chat/transformation.py index 61c7d51a889..9e168f14401 100644 --- a/litellm/llms/bedrock_mantle/chat/transformation.py +++ b/litellm/llms/bedrock_mantle/chat/transformation.py @@ -24,7 +24,7 @@ from litellm.llms.bedrock_mantle.common_utils import ( BEDROCK_MANTLE_DEFAULT_REGION, BedrockMantleAuthMixin, ) -from litellm.llms.openai.chat.gpt_5_transformation import get_effort_level, is_gpt_reasoning_series_name +from litellm.llms.openai.chat.gpt_reasoning_transformation import get_effort_level, is_gpt_reasoning_series_name from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams diff --git a/litellm/llms/chatgpt/authenticator.py b/litellm/llms/chatgpt/authenticator.py index ff9c6476cba..a5fb1c9c6b9 100644 --- a/litellm/llms/chatgpt/authenticator.py +++ b/litellm/llms/chatgpt/authenticator.py @@ -1,9 +1,9 @@ -import base64 import json import os import time from collections.abc import Mapping -from typing import Final, TypeAlias +from functools import lru_cache +from typing import TYPE_CHECKING, Final import httpx from pydantic import JsonValue, TypeAdapter, ValidationError @@ -24,19 +24,21 @@ from .common_utils import ( CHATGPT_OAUTH_TOKEN_URL, GetAccessTokenError, GetDeviceCodeError, + JsonObject, RefreshAccessTokenError, + decode_jwt_claims, + extract_chatgpt_account_id, ) +if TYPE_CHECKING: + from litellm.types.router import GenericLiteLLMParams + TOKEN_EXPIRY_SKEW_SECONDS: Final = 60 TOKEN_REFRESH_TIMEOUT_SECONDS: Final = 30 DEVICE_CODE_TIMEOUT_SECONDS: Final = 15 * 60 DEVICE_CODE_COOLDOWN_SECONDS: Final = 5 * 60 DEVICE_CODE_POLL_SLEEP_SECONDS: Final = 5 -OPENAI_AUTH_CLAIM_KEY: Final = "https://api.openai.com/auth" - -JsonObject: TypeAlias = Mapping[str, JsonValue] - _JSON_OBJECT_ADAPTER: Final = TypeAdapter(JsonObject) @@ -44,6 +46,26 @@ def _optional_str(value: JsonValue | None) -> str | None: return value if isinstance(value, str) else None +def get_chatgpt_auth_file( + litellm_params: "Mapping[str, object] | GenericLiteLLMParams | None", +) -> str | None: + if litellm_params is None: + return None + value: Final = ( + litellm_params.get("chatgpt_auth_file") + if isinstance(litellm_params, Mapping) + else litellm_params.chatgpt_auth_file + ) + if not isinstance(value, str) or not value: + return None + return os.path.abspath(os.path.expanduser(value)) + + +@lru_cache(maxsize=128) +def get_cached_authenticator(auth_file: str) -> "Authenticator": + return Authenticator(auth_file=auth_file) + + def _token_refresh_timeout() -> httpx.Timeout: configured: Final = get_configured_request_timeout() seconds: Final = ( @@ -53,13 +75,13 @@ def _token_refresh_timeout() -> httpx.Timeout: class Authenticator: - def __init__(self) -> None: - self.token_dir = os.getenv( - "CHATGPT_TOKEN_DIR", - os.path.expanduser("~/.config/litellm/chatgpt"), + def __init__(self, auth_file: str | None = None) -> None: + default_auth_file: Final = os.path.join( + os.getenv("CHATGPT_TOKEN_DIR", os.path.expanduser("~/.config/litellm/chatgpt")), + os.getenv("CHATGPT_AUTH_FILE", "auth.json"), ) - self.auth_file = os.path.join(self.token_dir, os.getenv("CHATGPT_AUTH_FILE", "auth.json")) - self._ensure_token_dir() + self.auth_file = os.path.abspath(os.path.expanduser(auth_file or default_auth_file)) + self.token_dir = os.path.dirname(self.auth_file) def get_api_base(self) -> str: return os.getenv("CHATGPT_API_BASE") or os.getenv("OPENAI_CHATGPT_API_BASE") or CHATGPT_API_BASE @@ -99,16 +121,18 @@ class Authenticator: tokens: Final = self._login_device_code() return tokens["access_token"] - def get_account_id(self) -> str | None: + def get_account_id(self, access_token: str | None = None) -> str | None: auth_data: Final = self._read_auth_file() + if access_token and (not auth_data or auth_data.get("access_token") != access_token): + return extract_chatgpt_account_id(access_token) if not auth_data: return None account_id: Final = _optional_str(auth_data.get("account_id")) if account_id: return account_id id_token: Final = auth_data.get("id_token") - access_token: Final = auth_data.get("access_token") - derived: Final = self._extract_account_id(_optional_str(id_token or access_token)) + saved_access_token: Final = auth_data.get("access_token") + derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token)) if derived: self._write_auth_file({**auth_data, "account_id": derived}) return derived @@ -129,6 +153,7 @@ class Authenticator: def _write_auth_file(self, data: JsonObject) -> None: try: + self._ensure_token_dir() with open(self.auth_file, "w") as f: json.dump(data, f) except OSError as exc: @@ -146,34 +171,11 @@ class Authenticator: return time.time() >= float(derived_expires_at) - TOKEN_EXPIRY_SKEW_SECONDS def _get_expires_at(self, token: str) -> int | None: - claims: Final = self._decode_jwt_claims(token) - exp: Final = claims.get("exp") + exp: Final = decode_jwt_claims(token).get("exp") if isinstance(exp, (int, float)): return int(exp) return None - def _decode_jwt_claims(self, token: str) -> JsonObject: - try: - parts: Final = token.split(".") - if len(parts) < 2: - return {} - payload_b64: Final = parts[1] + "=" * (-len(parts[1]) % 4) - payload_bytes: Final = base64.urlsafe_b64decode(payload_b64) - return _JSON_OBJECT_ADAPTER.validate_python(json.loads(payload_bytes.decode("utf-8"))) - except Exception: - return {} - - def _extract_account_id(self, token: str | None) -> str | None: - if not token: - return None - claims: Final = self._decode_jwt_claims(token) - auth_claims: Final = claims.get(OPENAI_AUTH_CLAIM_KEY) - if isinstance(auth_claims, dict): - account_id: Final = auth_claims.get("chatgpt_account_id") - if isinstance(account_id, str) and account_id: - return account_id - return None - def _login_device_code(self) -> dict[str, str]: cooldown_remaining: Final = self._get_device_code_cooldown_remaining(self._read_auth_file()) if cooldown_remaining > 0: @@ -361,15 +363,16 @@ class Authenticator: "refresh_token": _optional_str(data.get("refresh_token")) or refresh_token, "id_token": id_token, } + account_id: Final = self.get_account_id() auth_data: Final = self._build_auth_record(refreshed) - self._write_auth_file(auth_data) + self._write_auth_file({**auth_data, "account_id": account_id} if account_id else auth_data) return refreshed def _build_auth_record(self, tokens: dict[str, str]) -> JsonObject: access_token: Final = tokens.get("access_token") id_token: Final = tokens.get("id_token") expires_at: Final = self._get_expires_at(access_token) if access_token else None - account_id: Final = self._extract_account_id(id_token or access_token) + account_id: Final = extract_chatgpt_account_id(id_token or access_token) return { "access_token": access_token, "refresh_token": tokens.get("refresh_token"), diff --git a/litellm/llms/chatgpt/chat/transformation.py b/litellm/llms/chatgpt/chat/transformation.py index a1aac7f315b..9005185931f 100644 --- a/litellm/llms/chatgpt/chat/transformation.py +++ b/litellm/llms/chatgpt/chat/transformation.py @@ -1,14 +1,20 @@ +from collections.abc import Mapping from typing import Any, Final from litellm.exceptions import AuthenticationError from litellm.llms.openai.openai import OpenAIConfig from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import GenericLiteLLMParams -from ..authenticator import Authenticator +from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file from ..common_utils import ( GetAccessTokenError, + chatgpt_client_api_base, ensure_chatgpt_session_id, + get_chatgpt_client_credential, get_chatgpt_default_headers, + get_chatgpt_request_credential, + strip_chatgpt_client_credential_headers, ) from .streaming_utils import ChatGPTToolCallNormalizer @@ -19,9 +25,27 @@ class ChatGPTConfig(OpenAIConfig): api_key: str | None = None, api_base: str | None = None, custom_llm_provider: str = "openai", + authenticator: Authenticator | None = None, ) -> None: super().__init__() - self.authenticator = Authenticator() + self.authenticator = authenticator if authenticator is not None else Authenticator() + + def _resolve_authenticator( + self, litellm_params: Mapping[str, object] | GenericLiteLLMParams | None + ) -> Authenticator: + auth_file: Final = get_chatgpt_auth_file(litellm_params) + return get_cached_authenticator(auth_file) if auth_file else self.authenticator + + @staticmethod + def _get_access_token_or_raise(authenticator: Authenticator, model: str, llm_provider: str) -> str: + try: + return authenticator.get_access_token() + except GetAccessTokenError as e: + raise AuthenticationError( + model=model, + llm_provider=llm_provider, + message=str(e), + ) def api_base_without_login(self) -> str: return self.authenticator.get_api_base() @@ -32,16 +56,16 @@ class ChatGPTConfig(OpenAIConfig): api_base: str | None, api_key: str | None, custom_llm_provider: str, + litellm_params: Mapping[str, object] | GenericLiteLLMParams | None = None, ) -> tuple[str | None, str | None, str]: - dynamic_api_base: Final = self.api_base_without_login() - try: - dynamic_api_key: Final = self.authenticator.get_access_token() - except GetAccessTokenError as e: - raise AuthenticationError( - model=model, - llm_provider=custom_llm_provider, - message=str(e), - ) + client_credential: Final = get_chatgpt_request_credential(litellm_params) + if client_credential is not None: + return chatgpt_client_api_base(api_base), client_credential.access_token, custom_llm_provider + if api_key: + return api_base or self.api_base_without_login(), api_key, custom_llm_provider + authenticator: Final = self._resolve_authenticator(litellm_params) + dynamic_api_base: Final = authenticator.get_api_base() + dynamic_api_key: Final = self._get_access_token_or_raise(authenticator, model, custom_llm_provider) return dynamic_api_base, dynamic_api_key, custom_llm_provider def get_openai_compatible_provider_info( @@ -50,26 +74,49 @@ class ChatGPTConfig(OpenAIConfig): api_base: str | None, api_key: str | None, custom_llm_provider: str, + litellm_params: Mapping[str, object] | GenericLiteLLMParams | None = None, ) -> tuple[str | None, str | None, str]: - return self._get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider) + return self._get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider, litellm_params) def validate_environment( self, - headers: dict, + headers: dict[str, str], model: str, messages: list[AllMessageValues], optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict[str, str]: + client_credential: Final = get_chatgpt_client_credential(headers) + if client_credential is not None: + return dict( + get_chatgpt_default_headers( + client_credential.access_token, + client_credential.account_id, + ensure_chatgpt_session_id(litellm_params), + ), + **strip_chatgpt_client_credential_headers(headers), + ) + authenticator: Final = self._resolve_authenticator(litellm_params) + resolved_api_key: Final = ( + self._get_access_token_or_raise(authenticator, model, "chatgpt") + if get_chatgpt_auth_file(litellm_params) + else api_key + ) validated_headers: Final = super().validate_environment( - headers, model, messages, optional_params, litellm_params, api_key, api_base + dict(strip_chatgpt_client_credential_headers(headers)), + model, + messages, + optional_params, + litellm_params, + resolved_api_key, + api_base, ) - account_id: Final = self.authenticator.get_account_id() + account_id: Final = authenticator.get_account_id(resolved_api_key) session_id: Final = ensure_chatgpt_session_id(litellm_params) - default_headers: Final = get_chatgpt_default_headers(api_key or "", account_id, session_id) + default_headers: Final = get_chatgpt_default_headers(resolved_api_key or "", account_id, session_id) return {**default_headers, **validated_headers} def post_stream_processing(self, stream: Any) -> ChatGPTToolCallNormalizer: diff --git a/litellm/llms/chatgpt/common_utils.py b/litellm/llms/chatgpt/common_utils.py index fe33219f110..ec7fbfa92da 100644 --- a/litellm/llms/chatgpt/common_utils.py +++ b/litellm/llms/chatgpt/common_utils.py @@ -2,14 +2,32 @@ Constants and helpers for ChatGPT subscription OAuth. """ +import base64 +import json import os import platform -from typing import Any, Final +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeAlias +from urllib.parse import urlsplit from uuid import uuid4 import httpx +from pydantic import JsonValue, TypeAdapter +from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.utils import ProviderSpecificHeader + +if TYPE_CHECKING: + from litellm.types.router import GenericLiteLLMParams + +_SCOPED_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]] = TypeAdapter( + ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None +) +_STRING_HEADERS: Final = TypeAdapter(Mapping[str, str]) +_PARAMS: Final = TypeAdapter(Mapping[str, object]) # OAuth + API constants (derived from openai/codex) CHATGPT_AUTH_BASE: Final = "https://auth.openai.com" @@ -19,6 +37,12 @@ CHATGPT_OAUTH_TOKEN_URL: Final = f"{CHATGPT_AUTH_BASE}/oauth/token" CHATGPT_DEVICE_VERIFY_URL: Final = f"{CHATGPT_AUTH_BASE}/codex/device" CHATGPT_API_BASE: Final = "https://chatgpt.com/backend-api/codex" CHATGPT_CLIENT_ID: Final = "app_EMoamEEZ73f0CkXaXp7hrann" +CHATGPT_ACCOUNT_ID_HEADER: Final = "ChatGPT-Account-Id" +OPENAI_AUTH_CLAIM_KEY: Final = "https://api.openai.com/auth" + +JsonObject: TypeAlias = Mapping[str, JsonValue] +_JSON_OBJECT_ADAPTER: Final = TypeAdapter(JsonObject) +_EMPTY_CLAIMS: Final[JsonObject] = MappingProxyType({}) DEFAULT_ORIGINATOR: Final = "codex_cli_rs" DEFAULT_USER_AGENT: Final = "codex_cli_rs/0.0.0 (Unknown 0; unknown) unknown" @@ -229,7 +253,7 @@ def get_chatgpt_default_headers( access_token: str, account_id: str | None, session_id: str | None = None, -) -> dict: +) -> dict[str, str]: originator: Final = get_chatgpt_originator() user_agent: Final = get_chatgpt_user_agent(originator) headers: Final = { @@ -242,30 +266,133 @@ def get_chatgpt_default_headers( if session_id: headers["session_id"] = session_id if account_id: - headers["ChatGPT-Account-Id"] = account_id + headers[CHATGPT_ACCOUNT_ID_HEADER] = account_id return headers +def decode_jwt_claims(token: str) -> JsonObject: + try: + parts: Final = token.split(".") + if len(parts) < 2: + return _EMPTY_CLAIMS + payload_b64: Final = parts[1] + "=" * (-len(parts[1]) % 4) + payload_bytes: Final = base64.urlsafe_b64decode(payload_b64) + return _JSON_OBJECT_ADAPTER.validate_python(json.loads(payload_bytes.decode("utf-8"))) + except Exception: + return _EMPTY_CLAIMS + + +def extract_chatgpt_account_id(token: str | None) -> str | None: + if not token: + return None + auth_claims: Final = decode_jwt_claims(token).get(OPENAI_AUTH_CLAIM_KEY) + if not isinstance(auth_claims, dict): + return None + account_id: Final = auth_claims.get("chatgpt_account_id") + return account_id if isinstance(account_id, str) and account_id else None + + +def _bearer_token(value: str) -> str: + scheme, separator, token = value.strip().partition(" ") + if not separator: + return scheme + return token.strip() if scheme.lower() == "bearer" else "" + + +def is_chatgpt_oauth_key(value: str | None) -> bool: + if value is None: + return False + return OPENAI_AUTH_CLAIM_KEY in decode_jwt_claims(_bearer_token(value)) + + +@dataclass(frozen=True, slots=True) +class ChatGPTClientCredential: + access_token: str + account_id: str | None + + +def _header_value(headers: Mapping[str, str], name: str) -> str | None: + return next((value for key, value in headers.items() if key.lower() == name), None) + + +def get_chatgpt_client_credential(headers: Mapping[str, str]) -> ChatGPTClientCredential | None: + auth_header: Final = _header_value(headers, "authorization") + if auth_header is None or not is_chatgpt_oauth_key(auth_header): + return None + access_token: Final = _bearer_token(auth_header) + account_id: Final = _header_value(headers, CHATGPT_ACCOUNT_ID_HEADER.lower()) or extract_chatgpt_account_id( + access_token + ) + return ChatGPTClientCredential(access_token=access_token, account_id=account_id) + + +def select_chatgpt_client_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]: + if get_chatgpt_client_credential(headers) is None: + return MappingProxyType({}) + return MappingProxyType( + { + key: value + for key, value in headers.items() + if (key.lower() == "authorization" and is_chatgpt_oauth_key(value)) + or key.lower() == CHATGPT_ACCOUNT_ID_HEADER.lower() + } + ) + + +def strip_chatgpt_client_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]: + return MappingProxyType( + { + key: value + for key, value in headers.items() + if key.lower() not in ("authorization", CHATGPT_ACCOUNT_ID_HEADER.lower()) + } + ) + + +def get_chatgpt_request_credential( + params: "Mapping[str, object] | GenericLiteLLMParams | None", +) -> ChatGPTClientCredential | None: + normalized: Final = _normalize_litellm_params(params) + scoped: Final = normalized.get("provider_specific_header") + extra: Final = normalized.get("extra_headers") + headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=_SCOPED_HEADERS.validate_python(scoped), custom_llm_provider="chatgpt" + ) + return get_chatgpt_client_credential(headers) or get_chatgpt_client_credential( + _STRING_HEADERS.validate_python(extra) if extra is not None else MappingProxyType({}) + ) + + +def chatgpt_client_api_base(api_base: str | None) -> str: + base: Final = api_base or CHATGPT_API_BASE + parsed: Final = urlsplit(base) + if ( + parsed.scheme != "https" + or parsed.hostname != "chatgpt.com" + or parsed.port not in (None, 443) + or parsed.username + or parsed.password + ): + raise ChatGPTAuthError( + status_code=400, message="Client ChatGPT credentials require the official HTTPS ChatGPT endpoint" + ) + return base + + def get_chatgpt_default_instructions() -> str: return os.getenv("CHATGPT_DEFAULT_INSTRUCTIONS") or CHATGPT_DEFAULT_INSTRUCTIONS -def _normalize_litellm_params(litellm_params: Any | None) -> dict: - if litellm_params is None: - return {} - if isinstance(litellm_params, dict): - return litellm_params - if hasattr(litellm_params, "model_dump"): - try: - return litellm_params.model_dump() - except Exception: - return {} - if hasattr(litellm_params, "dict"): - try: - return litellm_params.dict() - except Exception: - return {} - return {} +def _normalize_litellm_params(litellm_params: object) -> Mapping[str, object]: + if isinstance(litellm_params, Mapping): + return _PARAMS.validate_python(litellm_params) + serializer: Final = getattr(litellm_params, "model_dump", None) or getattr(litellm_params, "dict", None) + if not callable(serializer): + return MappingProxyType({}) + try: + return _PARAMS.validate_python(serializer()) + except Exception: + return MappingProxyType({}) def get_chatgpt_session_id(litellm_params: object) -> str | None: @@ -274,8 +401,8 @@ def get_chatgpt_session_id(litellm_params: object) -> str | None: value = params.get(key) if value: return str(value) - metadata: Final = params.get("metadata") - if isinstance(metadata, dict): + metadata: Final = _normalize_litellm_params(params.get("metadata")) + if metadata: value = metadata.get("session_id") if value: return str(value) diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 34953a79c60..2352de35b5b 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -25,13 +25,15 @@ from litellm.types.llms.openai import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders -from ..authenticator import Authenticator +from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file from ..common_utils import ( CHATGPT_API_BASE, GetAccessTokenError, ensure_chatgpt_session_id, + get_chatgpt_client_credential, get_chatgpt_default_headers, get_chatgpt_default_instructions, + strip_chatgpt_client_credential_headers, ) if TYPE_CHECKING: @@ -49,14 +51,35 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.CHATGPT + def _resolve_authenticator(self, litellm_params: GenericLiteLLMParams | None) -> Authenticator: + auth_file: Final = get_chatgpt_auth_file(litellm_params) + return get_cached_authenticator(auth_file) if auth_file else self.authenticator + def validate_environment( self, - headers: dict, + headers: dict[str, str], model: str, litellm_params: GenericLiteLLMParams | None, - ) -> dict: + ) -> dict[str, str]: + session_id: Final = ensure_chatgpt_session_id(litellm_params) + client_credential: Final = get_chatgpt_client_credential(headers) + if client_credential is not None: + return { + **get_chatgpt_default_headers(client_credential.access_token, client_credential.account_id, session_id), + **strip_chatgpt_client_credential_headers(headers), + } + authenticator: Final = self._resolve_authenticator(litellm_params) + safe_headers: Final = strip_chatgpt_client_credential_headers(headers) + configured_api_key: Final = litellm_params.api_key if litellm_params is not None else None + if configured_api_key and not get_chatgpt_auth_file(litellm_params): + return { + **get_chatgpt_default_headers( + configured_api_key, authenticator.get_account_id(configured_api_key), session_id + ), + **safe_headers, + } try: - access_token: Final = self.authenticator.get_access_token() + access_token: Final = authenticator.get_access_token() except GetAccessTokenError as e: raise AuthenticationError( model=model, @@ -64,10 +87,9 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): message=str(e), ) - account_id: Final = self.authenticator.get_account_id() - session_id: Final = ensure_chatgpt_session_id(litellm_params) + account_id: Final = authenticator.get_account_id() default_headers: Final = get_chatgpt_default_headers(access_token, account_id, session_id) - return {**default_headers, **headers} + return {**default_headers, **safe_headers} def transform_responses_api_request( self, @@ -79,7 +101,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): ) -> dict: request: Final = super().transform_responses_api_request( model, - input, + [{"role": "user", "content": input}] if isinstance(input, str) else input, response_api_optional_request_params, litellm_params, headers, diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index cffeb44f12c..0b64885df45 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -20,6 +20,7 @@ from aiohttp import ClientSession, DummyCookieJar, TCPConnector from httpx import USE_CLIENT_DEFAULT, AsyncHTTPTransport, HTTPTransport from httpx._types import CertTypes, RequestFiles from httpx._utils import get_environment_proxies +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger @@ -566,6 +567,34 @@ async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> N raise MaskedHTTPStatusError(e, message=_text, text=_text) from None +_TIMEOUT_PHASES: Final[Mapping[type[httpx.TimeoutException], tuple[str, str]]] = MappingProxyType( + { + httpx.ConnectTimeout: ("Connection", "connect"), + httpx.ReadTimeout: ("Read", "read"), + httpx.WriteTimeout: ("Write", "write"), + httpx.PoolTimeout: ("Connection pool", "pool"), + } +) +_UNKNOWN_TIMEOUT_PHASE: Final = ("Request", "read") +_TIMEOUT_EXTENSION: Final = TypeAdapter(dict[str, float | None]) + + +def _timeout_phase_seconds(error: httpx.TimeoutException, phase: str) -> float | None: + """The limit httpx enforced for ``phase`` on the request that raised ``error``, from the request itself.""" + return _TIMEOUT_EXTENSION.validate_python(error.request.extensions["timeout"]).get(phase) + + +def _raise_timeout(error: httpx.TimeoutException, started_at: float) -> NoReturn: + label, phase = _TIMEOUT_PHASES.get(type(error), _UNKNOWN_TIMEOUT_PHASE) + seconds: Final = _timeout_phase_seconds(error, phase) + time_taken: Final = round(time.time() - started_at, 3) + raise litellm.Timeout( + message=f"{label} timed out. Timeout passed={seconds}, time taken={time_taken} seconds", + model="default-model-name", + llm_provider="litellm-httpx-handler", + ) + + class HTTPResponseLimitError(ValueError): pass @@ -870,20 +899,7 @@ class AsyncHTTPHandler: finally: await new_client.aclose() except httpx.TimeoutException as e: - end_time: Final = time.time() - time_delta: Final = round(end_time - start_time, 3) - headers = {} - error_response: Final[httpx.Response | None] = getattr(e, "response", None) - if error_response is not None: - for key, value in error_response.headers.items(): - headers[f"response_headers-{key}"] = value - - raise litellm.Timeout( - message=f"Connection timed out. Timeout passed={timeout}, time taken={time_delta} seconds", - model="default-model-name", - llm_provider="litellm-httpx-handler", - headers=headers, - ) + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: await _raise_masked_async_error(e, stream) except Exception as e: @@ -902,6 +918,7 @@ class AsyncHTTPHandler: follow_redirects: bool | None = None, ): _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT + start_time: Final = time.time() try: if timeout is None: timeout = self.timeout @@ -946,18 +963,7 @@ class AsyncHTTPHandler: finally: await new_client.aclose() except httpx.TimeoutException as e: - headers = {} - error_response: Final[httpx.Response | None] = getattr(e, "response", None) - if error_response is not None: - for key, value in error_response.headers.items(): - headers[f"response_headers-{key}"] = value - - raise litellm.Timeout( - message=f"Connection timed out after {timeout} seconds.", - model="default-model-name", - llm_provider="litellm-httpx-handler", - headers=headers, - ) + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: await _raise_masked_async_error(e, stream) except Exception as e: @@ -974,6 +980,7 @@ class AsyncHTTPHandler: stream: bool = False, content: _RequestContent | None = None, ): + start_time: Final = time.time() try: if timeout is None: timeout = self.timeout @@ -1012,18 +1019,7 @@ class AsyncHTTPHandler: finally: await new_client.aclose() except httpx.TimeoutException as e: - headers = {} - error_response: Final[httpx.Response | None] = getattr(e, "response", None) - if error_response is not None: - for key, value in error_response.headers.items(): - headers[f"response_headers-{key}"] = value - - raise litellm.Timeout( - message=f"Connection timed out after {timeout} seconds.", - model="default-model-name", - llm_provider="litellm-httpx-handler", - headers=headers, - ) + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: await _raise_masked_async_error(e, stream) except Exception as e: @@ -1514,6 +1510,7 @@ class HTTPHandler: content: _RequestContent | None = None, logging_obj: LiteLLMLoggingObject | None = None, ): + start_time: Final = time.time() try: # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) request_data, request_content = _prepare_request_data_and_content(data, content) @@ -1546,12 +1543,8 @@ class HTTPHandler: _anchor_handler_to(response, self) response.raise_for_status() return response - except httpx.TimeoutException: - raise litellm.Timeout( - message=f"Connection timed out after {timeout} seconds.", - model="default-model-name", - llm_provider="litellm-httpx-handler", - ) + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: _raise_masked_sync_error(e, stream) except Exception as e: @@ -1568,6 +1561,7 @@ class HTTPHandler: timeout: float | httpx.Timeout | None = None, content: _RequestContent | None = None, ): + start_time: Final = time.time() try: # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) request_data, request_content = _prepare_request_data_and_content(data, content) @@ -1598,12 +1592,8 @@ class HTTPHandler: _anchor_handler_to(response, self) response.raise_for_status() return response - except httpx.TimeoutException: - raise litellm.Timeout( - message=f"Connection timed out after {timeout} seconds.", - model="default-model-name", - llm_provider="litellm-httpx-handler", - ) + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: _raise_masked_sync_error(e, stream) except Exception as e: @@ -1620,6 +1610,7 @@ class HTTPHandler: timeout: float | httpx.Timeout | None = None, content: _RequestContent | None = None, ): + start_time: Final = time.time() try: # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) request_data, request_content = _prepare_request_data_and_content(data, content) @@ -1649,12 +1640,8 @@ class HTTPHandler: if stream: _anchor_handler_to(response, self) return response - except httpx.TimeoutException: - raise litellm.Timeout( - message=f"Connection timed out after {timeout} seconds.", - model="default-model-name", - llm_provider="litellm-httpx-handler", - ) + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: _raise_masked_sync_error(e, stream) except Exception as e: @@ -1671,6 +1658,7 @@ class HTTPHandler: stream: bool = False, content: _RequestContent | None = None, ): + start_time: Final = time.time() try: # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) request_data, request_content = _prepare_request_data_and_content(data, content) @@ -1701,12 +1689,8 @@ class HTTPHandler: _anchor_handler_to(response, self) response.raise_for_status() return response - except httpx.TimeoutException: - raise litellm.Timeout( - message=f"Connection timed out after {timeout} seconds.", - model="default-model-name", - llm_provider="litellm-httpx-handler", - ) + except httpx.TimeoutException as e: + _raise_timeout(e, start_time) except httpx.HTTPStatusError as e: _raise_masked_sync_error(e, stream) except Exception as e: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 1b491f1614d..7b808cf2be2 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -198,6 +198,26 @@ from litellm.utils import ( ) +def _merge_headers_case_insensitive( + headers: Mapping[str, object], extra_headers: Mapping[str, object] | None +) -> dict[str, str]: # mutable-ok: legacy HTTP handler methods require a dict of headers + overridden: Final = frozenset(name.lower() for name in extra_headers or {}) + return { + **{name: str(value) for name, value in headers.items() if name.lower() not in overridden}, + **{name: str(value) for name, value in (extra_headers or {}).items()}, + } + + +def _client_request_headers( + response_api_optional_request_params: Mapping[str, object], + extra_headers: Mapping[str, object] | None, +) -> dict[str, str]: # mutable-ok: provider validate_environment methods require a dict + from litellm.litellm_core_utils.core_helpers import as_str_mapping + + optional_headers: Final = as_str_mapping(response_api_optional_request_params.get("extra_headers")) + return _merge_headers_case_insensitive(optional_headers or {}, extra_headers) + + def _rust_responses_websocket_enabled( custom_llm_provider: str | None, ) -> bool: @@ -2906,14 +2926,18 @@ class BaseLLMHTTPHandler: else: sync_httpx_client = client - headers = responses_api_provider_config.validate_environment( - headers=response_api_optional_request_params.get("extra_headers", {}) or {}, - model=model, - litellm_params=litellm_params, - ) - - if extra_headers: - headers.update(extra_headers) + headers = { # rebind-ok: validation replaces the method's incoming headers + **responses_api_provider_config.validate_environment( + headers=( + _client_request_headers(response_api_optional_request_params, extra_headers) + if custom_llm_provider == "chatgpt" + else response_api_optional_request_params.get("extra_headers", {}) or {} + ), + model=model, + litellm_params=litellm_params, + ), + **((extra_headers or {}) if custom_llm_provider != "chatgpt" else {}), + } # Check if streaming is requested stream = response_api_optional_request_params.get("stream", False) @@ -3104,11 +3128,15 @@ class BaseLLMHTTPHandler: validated_headers: Final = { **await asyncio.to_thread( responses_api_provider_config.validate_environment, - headers=response_api_optional_request_params.get("extra_headers", {}) or {}, + headers=( + _client_request_headers(response_api_optional_request_params, extra_headers) + if custom_llm_provider == "chatgpt" + else response_api_optional_request_params.get("extra_headers", {}) or {} + ), model=model, litellm_params=litellm_params, ), - **(extra_headers or {}), + **((extra_headers or {}) if custom_llm_provider != "chatgpt" else {}), } # Check if streaming is requested diff --git a/litellm/llms/elevenlabs/text_to_speech/transformation.py b/litellm/llms/elevenlabs/text_to_speech/transformation.py index f910ef8c057..23ccf41c9da 100644 --- a/litellm/llms/elevenlabs/text_to_speech/transformation.py +++ b/litellm/llms/elevenlabs/text_to_speech/transformation.py @@ -56,9 +56,9 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig): # Response format mappings from OpenAI to ElevenLabs FORMAT_MAPPINGS = { "mp3": "mp3_44100_128", - "pcm": "pcm_44100", + "pcm": "pcm_24000", "opus": "opus_48000_128", - # ElevenLabs does not support WAV, AAC, or FLAC formats. + "wav": "wav_24000", } ELEVENLABS_QUERY_PARAMS_KEY = "__elevenlabs_query_params__" diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_reasoning_transformation.py similarity index 100% rename from litellm/llms/openai/chat/gpt_5_transformation.py rename to litellm/llms/openai/chat/gpt_reasoning_transformation.py diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 73f40f5e0e4..9d4df354a30 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -48,7 +48,7 @@ from litellm.utils import ( from ...types.llms.openai import * from ..base import BaseLLM -from .chat.gpt_5_transformation import OpenAIGPT5Config +from .chat.gpt_reasoning_transformation import OpenAIGPT5Config from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig from .chat.o_series_transformation import OpenAIOSeriesConfig from .common_utils import ( diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index b90f6aa01da..07822bb17b3 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -25,7 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig -from litellm.llms.openai.chat.gpt_5_transformation import ( +from litellm.llms.openai.chat.gpt_reasoning_transformation import ( is_gpt_reasoning_series_name, unsupported_reasoning_effort_message, ) @@ -291,7 +291,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): level the row cannot gate because the same model takes it here and refuses it on chat. Azure overrides this so a bare deployment name reads the azure/ entry. """ - from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config + from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config return OpenAIGPT5Config.row_disables_reasoning_effort_level(model, level) @@ -305,7 +305,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): copy of the rule here is what let this surface keep forwarding temperature after the chat surface stopped. """ - from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config + from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config return OpenAIGPT5Config.effort_resolves_to_none(model, effort) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 36337fddbdc..115d914d3ac 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -249,5 +249,13 @@ "/v1/responses", "/v1/messages" ] + }, + "refuseless": { + "base_url": "https://api.refuseless.com/v1", + "api_key_env": "REFUSELESS_API_KEY", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ] } } diff --git a/litellm/llms/pass_through/guardrail_translation/handler.py b/litellm/llms/pass_through/guardrail_translation/handler.py index 3ce8d7ddad5..dea8bcb73a4 100644 --- a/litellm/llms/pass_through/guardrail_translation/handler.py +++ b/litellm/llms/pass_through/guardrail_translation/handler.py @@ -90,7 +90,7 @@ class PassThroughEndpointHandler(BaseTranslation): payload_to_check: Final = { k: v for k, v in without_server_streaming_classification(data).items() - if not k.startswith("_") and k not in ("metadata", "litellm_logging_obj") + if not k.startswith("_") and k not in ("metadata", "litellm_metadata", "litellm_logging_obj") } verbose_proxy_logger.debug("PassThroughEndpointHandler: Using full payload for guardrail") return safe_dumps(payload_to_check) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 71bb05ee155..ed86934db63 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -726,6 +726,9 @@ def gemini_convert_messages_with_history( from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig forward_function_call_id: Final = VertexGeminiConfig.forward_gemini_function_call_id(model or "") + supports_multimodal_function_responses: Final = model is None or ( + VertexGeminiConfig.supports_multimodal_function_responses(model) + ) try: while msg_i < len(messages): @@ -1029,6 +1032,7 @@ def gemini_convert_messages_with_history( messages[msg_i], last_message_with_tool_calls, forward_function_call_id=forward_function_call_id, + supports_multimodal_function_responses=supports_multimodal_function_responses, ) msg_i += 1 # Handle both single part and list of parts (for Computer Use with images) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index d788ccb0f1d..9acbc9c459c 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -689,6 +689,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) -> bool: return cls._forward_gemini_function_call_id(model) + @classmethod + def supports_multimodal_function_responses(cls, model: str) -> bool: + return cls.is_gemini_3_or_newer(model) + def _supports_penalty_parameters(self, model: str) -> bool: # Gemini 3 models do not support penalty parameters if VertexGeminiConfig._is_gemini_3_or_newer(model): diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/__init__.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/__init__.py index a9c1e5819f2..991d7e8f1bc 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/__init__.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/__init__.py @@ -12,7 +12,7 @@ def get_vertex_ai_partner_model_config(model: str, vertex_publisher_or_api_spec: return VertexAIAi21Config() elif vertex_publisher_or_api_spec == "openapi" or vertex_publisher_or_api_spec == "mistralai": - from .llama3.transformation import VertexAILlama3Config + from .llama.transformation import VertexAILlama3Config return VertexAILlama3Config() else: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama/transformation.py similarity index 100% rename from litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py rename to litellm/llms/vertex_ai/vertex_ai_partner_models/llama/transformation.py diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 6cff9729189..0cdd92bb071 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -7,12 +7,14 @@ Handles Authentication and generating request urls for Vertex AI and Google AI S import asyncio import json import threading +from collections import OrderedDict from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, Literal, Protocol from urllib.parse import urlparse import litellm from litellm._logging import verbose_logger +from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -70,16 +72,20 @@ class _VertexCredentialsObject(Protocol): def refresh(self, request: object) -> None: ... +_VertexCredentialCacheKey = tuple[str | None, str | None] + + class VertexBase: def __init__(self) -> None: super().__init__() self.access_token: str | None = None self.refresh_token: str | None = None self._credentials: GoogleCredentialsObject | None = None - self._credentials_project_mapping: dict[ - tuple[VERTEX_CREDENTIALS_TYPES | None, str | None], + self._credentials_project_mapping: OrderedDict[ + _VertexCredentialCacheKey, tuple[_VertexCredentialsObject, str | None], - ] = {} + ] = OrderedDict() + self._credentials_cache_lock = threading.RLock() self.project_id: str | None = None self.async_handler: AsyncHTTPHandler | None = None # Per-credential-key asyncio.Lock for single-flight async refresh. @@ -91,10 +97,10 @@ class VertexBase: # each lock; the entry is pruned when the count reaches zero, so the # dict stays bounded even in long-running high-cardinality deployments # without depending on any private asyncio internals. - self._async_refresh_locks: dict[tuple, asyncio.Lock] = {} - self._async_refresh_lock_refcounts: dict[tuple, int] = {} + self._async_refresh_locks: dict[_VertexCredentialCacheKey, asyncio.Lock] = {} + self._async_refresh_lock_refcounts: dict[_VertexCredentialCacheKey, int] = {} # Tracks in-flight background refresh tasks to avoid duplicate refreshes. - self._background_refresh_tasks: dict[tuple, asyncio.Task] = {} + self._background_refresh_tasks: dict[_VertexCredentialCacheKey, asyncio.Task] = {} # Protects the sync get_access_token refresh path. # Use RLock so that the reauthentication retry path (which calls # back into get_access_token while still holding the lock) can @@ -378,7 +384,7 @@ class VertexBase: with self._sync_refresh_lock: credentials.refresh(Request()) - def _acquire_async_refresh_lock(self, credential_cache_key: tuple) -> asyncio.Lock: + def _acquire_async_refresh_lock(self, credential_cache_key: _VertexCredentialCacheKey) -> asyncio.Lock: """Increment the refcount and return the lock for ``credential_cache_key``. Every call must be paired with ``_release_async_refresh_lock`` once the @@ -391,7 +397,7 @@ class VertexBase: ) return lock - def _release_async_refresh_lock(self, credential_cache_key: tuple, lock: asyncio.Lock) -> None: + def _release_async_refresh_lock(self, credential_cache_key: _VertexCredentialCacheKey, lock: asyncio.Lock) -> None: """Decrement the refcount and drop the lock entry when it reaches zero. Must be called only after the caller has released ``lock`` (i.e. once @@ -409,7 +415,7 @@ class VertexBase: def _try_get_cached_token( self, - credential_cache_key: tuple, + credential_cache_key: _VertexCredentialCacheKey, project_id: str | None, ) -> tuple[str, str] | None: """ @@ -432,7 +438,7 @@ class VertexBase: def _try_get_usable_cached_token( self, - credential_cache_key: tuple, + credential_cache_key: _VertexCredentialCacheKey, project_id: str | None, ) -> tuple[str, str, "TokenState", _VertexCredentialsObject, str | None] | None: """ @@ -458,19 +464,37 @@ class VertexBase: return creds.token, resolved_project, token_state, creds, cached_project_id def _unpack_cached_credentials( - self, credential_cache_key: tuple + self, credential_cache_key: _VertexCredentialCacheKey ) -> tuple[_VertexCredentialsObject | None, str | None]: """ Return (credentials, project_id) from the cache, or (None, None) if not cached. Handles both tuple and legacy cache formats. """ - if credential_cache_key not in self._credentials_project_mapping: - return None, None - cached_entry: Final = self._credentials_project_mapping[credential_cache_key] + with self._credentials_cache_lock: + cached_entry: Final = self._credentials_project_mapping.get(credential_cache_key) + if cached_entry is None: + return None, None + self._credentials_project_mapping.move_to_end(credential_cache_key) if isinstance(cached_entry, tuple): return cached_entry return cached_entry, cached_entry.quota_project_id or getattr(cached_entry, "project_id", None) + def _cache_credentials( + self, + credential_cache_key: _VertexCredentialCacheKey, + credentials: _VertexCredentialsObject, + project_id: str | None, + *, + replace: bool = True, + ) -> None: + with self._credentials_cache_lock: + if not replace and credential_cache_key in self._credentials_project_mapping: + return + self._credentials_project_mapping[credential_cache_key] = (credentials, project_id) + self._credentials_project_mapping.move_to_end(credential_cache_key) + while len(self._credentials_project_mapping) > max(0, DEFAULT_MAX_LRU_CACHE_SIZE): + self._credentials_project_mapping.popitem(last=False) + def _get_token_state(self, credentials: _VertexCredentialsObject) -> "TokenState": """ Return the token state using google-auth's TokenState enum. @@ -494,7 +518,7 @@ class VertexBase: self, credentials: VERTEX_CREDENTIALS_TYPES | None, project_id: str | None, - credential_cache_key: tuple, + credential_cache_key: _VertexCredentialCacheKey, ) -> tuple[_VertexCredentialsObject, str | None]: """Load credentials via load_auth (in thread) and cache the result.""" try: @@ -507,17 +531,12 @@ class VertexBase: raise if _credentials is None: raise ValueError("Could not resolve credentials") - self._credentials_project_mapping[credential_cache_key] = ( - _credentials, - credential_project_id, - ) + self._cache_credentials(credential_cache_key, _credentials, credential_project_id) return _credentials, credential_project_id async def _background_refresh_credentials( self, credentials: _VertexCredentialsObject, - credential_cache_key: tuple, - credential_project_id: str | None, ) -> None: """ Refresh credentials in the background without blocking the calling request. @@ -528,25 +547,13 @@ class VertexBase: try: verbose_logger.debug("Background proactive credential refresh") await asyncify(self.refresh_auth)(credentials) - # Only update the cache if it still points at the credentials - # object we just refreshed. The per-key async lock is not held - # here, so a concurrent INVALID path may have already replaced - # this entry (e.g. via _handle_reauthentication_async, which - # creates a fresh credentials object). In that case our write - # would clobber the newer entry with a stale reference. - cached_creds, _ = self._unpack_cached_credentials(credential_cache_key) - if cached_creds is credentials: - self._credentials_project_mapping[credential_cache_key] = ( - credentials, - credential_project_id, - ) except Exception: verbose_logger.debug( "Background credential refresh failed, will retry on next request", exc_info=True, ) - async def _await_in_flight_background_refresh(self, credential_cache_key: tuple) -> None: + async def _await_in_flight_background_refresh(self, credential_cache_key: _VertexCredentialCacheKey) -> None: """Wait for an in-flight background refresh to finish, if any. google-auth's ``Credentials.refresh()`` is not safe to invoke @@ -568,8 +575,7 @@ class VertexBase: def _schedule_background_refresh( self, credentials: _VertexCredentialsObject, - credential_cache_key: tuple, - credential_project_id: str | None, + credential_cache_key: _VertexCredentialCacheKey, ) -> None: """Kick off a single background refresh for ``credential_cache_key``. @@ -581,9 +587,7 @@ class VertexBase: if existing is not None and not existing.done(): return self._background_refresh_tasks.pop(credential_cache_key, None) - task: Final = asyncio.create_task( - self._background_refresh_credentials(credentials, credential_cache_key, credential_project_id) - ) + task: Final = asyncio.create_task(self._background_refresh_credentials(credentials)) def _drop_background_refresh_task(_fut: asyncio.Future[None]) -> None: if self._background_refresh_tasks.get(credential_cache_key) is _fut: @@ -796,7 +800,7 @@ class VertexBase: self, credentials: VERTEX_CREDENTIALS_TYPES | None, project_id: str | None, - credential_cache_key: tuple, + credential_cache_key: _VertexCredentialCacheKey, error: Exception, ) -> tuple[str, str]: """ @@ -822,8 +826,8 @@ class VertexBase: ) # Clear the cached credentials - if credential_cache_key in self._credentials_project_mapping: - del self._credentials_project_mapping[credential_cache_key] + with self._credentials_cache_lock: + self._credentials_project_mapping.pop(credential_cache_key, None) # Retry once with _retry_reauth=True to prevent infinite recursion try: @@ -846,7 +850,7 @@ class VertexBase: self, credentials: VERTEX_CREDENTIALS_TYPES | None, project_id: str | None, - credential_cache_key: tuple, + credential_cache_key: _VertexCredentialCacheKey, error: Exception, ) -> tuple[str, str]: """ @@ -856,7 +860,8 @@ class VertexBase: "Handling async reauthentication for project_id: %s. Clearing cache and retrying once.", project_id ) - self._credentials_project_mapping.pop(credential_cache_key, None) + with self._credentials_cache_lock: + self._credentials_project_mapping.pop(credential_cache_key, None) try: ( @@ -875,10 +880,7 @@ class VertexBase: # references the OLD credentials object we just replaced, and # leaving it would force the next request to do a redundant # refresh/reauth before realizing the cached creds are stale. - self._credentials_project_mapping[resolved_cache_key] = ( - _credentials, - credential_project_id, - ) + self._cache_credentials(resolved_cache_key, _credentials, credential_project_id) if _credentials.token is None or not isinstance(_credentials.token, str): raise ValueError( @@ -924,21 +926,11 @@ class VertexBase: # Convert dict credentials to string for caching cache_credentials: Final = json.dumps(credentials) if isinstance(credentials, dict) else credentials credential_cache_key: Final = (cache_credentials, project_id) - _credentials: _VertexCredentialsObject | None = None - verbose_logger.debug("Checking cached credentials for project_id: %s", project_id) - if credential_cache_key in self._credentials_project_mapping: + _credentials, credential_project_id = self._unpack_cached_credentials(credential_cache_key) + if _credentials is not None: verbose_logger.debug("Cached credentials found for project_id: %s.", project_id) - # Retrieve both credentials and cached project_id - cached_entry: Final = self._credentials_project_mapping[credential_cache_key] - verbose_logger.debug("cached_entry: %s", cached_entry) - if isinstance(cached_entry, tuple): - _credentials, credential_project_id = cached_entry - else: - # Backward compatibility with old cache format - _credentials = cached_entry - credential_project_id = _credentials.quota_project_id or getattr(_credentials, "project_id", None) verbose_logger.debug( "Using cached credentials for project_id: %s", credential_project_id, @@ -963,10 +955,7 @@ class VertexBase: f"Could not resolve credentials - either dynamically or from environment, for project_id: {project_id}" ) # Cache the project_id and credentials from load_auth result (resolved project_id) - self._credentials_project_mapping[credential_cache_key] = ( - _credentials, - credential_project_id, - ) + self._cache_credentials(credential_cache_key, _credentials, credential_project_id) ## VALIDATE CREDENTIALS verbose_logger.debug("Validating credentials") @@ -974,11 +963,7 @@ class VertexBase: project_id = credential_project_id # Update cache with resolved project_id for future lookups resolved_cache_key: Final = (cache_credentials, project_id) - if resolved_cache_key not in self._credentials_project_mapping: - self._credentials_project_mapping[resolved_cache_key] = ( - _credentials, - credential_project_id, - ) + self._cache_credentials(resolved_cache_key, _credentials, credential_project_id, replace=False) # Check if credentials are None before accessing attributes if _credentials is None: @@ -991,10 +976,6 @@ class VertexBase: try: verbose_logger.debug("Credentials expired, refreshing") self.refresh_auth(_credentials) - self._credentials_project_mapping[credential_cache_key] = ( - _credentials, - credential_project_id, - ) except Exception as e: # if refresh fails, it's possible the user has re-authenticated via `gcloud auth application-default login` # in this case, we should try to reload the credentials by clearing the cache and retrying @@ -1043,9 +1024,9 @@ class VertexBase: # callers on the lock just to schedule that refresh. usable: Final = self._try_get_usable_cached_token(credential_cache_key, project_id) if usable is not None: - cached_token, resolved_project, token_state, creds, cached_project_id = usable + cached_token, resolved_project, token_state, creds, _cached_project_id = usable if token_state == TokenState.STALE: - self._schedule_background_refresh(creds, credential_cache_key, cached_project_id) + self._schedule_background_refresh(creds, credential_cache_key) return cached_token, resolved_project # === SLOW PATH (per-key lock) === @@ -1075,10 +1056,7 @@ class VertexBase: # reauth that only repopulated the unresolved key), which # would force the next request through an unnecessary # refresh/reauth cycle. - self._credentials_project_mapping[resolved_cache_key] = ( - _credentials, - credential_project_id, - ) + self._cache_credentials(resolved_cache_key, _credentials, credential_project_id) # Use google-auth's token_state to decide refresh strategy: # - STALE: token is usable but within REFRESH_THRESHOLD (3:45) of @@ -1098,7 +1076,6 @@ class VertexBase: self._schedule_background_refresh( _credentials, credential_cache_key, - credential_project_id, ) return current_token, project_id @@ -1117,10 +1094,6 @@ class VertexBase: try: verbose_logger.debug("Credentials expired, refreshing") await asyncify(self.refresh_auth)(_credentials) - self._credentials_project_mapping[credential_cache_key] = ( - _credentials, - credential_project_id, - ) except Exception as e: if "Reauthentication is needed" in str(e): verbose_logger.debug("Reauthentication needed, clearing cache and retrying") diff --git a/litellm/main.py b/litellm/main.py index f30d93361e5..23eac8ce0aa 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -130,7 +130,7 @@ from litellm.llms.bedrock.common_utils import ( from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled -from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config from litellm.llms.openai.common_utils import get_openai_credentials from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.vertex_ai.common_utils import ( @@ -237,7 +237,7 @@ from .llms.azure_ai.anthropic.transformation import azure_anthropic_messages_url from .llms.azure_ai.embed import AzureAIEmbedding from .llms.bedrock.chat import BedrockConverseLLM from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig -from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeConfig +from .llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import AmazonAnthropicClaudeConfig from .llms.bedrock.chat.native_output_fallback import NativeOutputAttemptResult, call_with_native_output_fallback from .llms.bedrock.embed.embedding import BedrockEmbedding from .llms.bedrock.image_edit.handler import BedrockImageEdit @@ -677,7 +677,11 @@ async def acompletion( } if custom_llm_provider is None: _supplemental: Final = cast( # cast-ok: kwargs values are request-scoped objects - "dict[str, object]", {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs} + "dict[str, object]", + { + **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs}, + "extra_headers": extra_headers, + }, ) _, custom_llm_provider, _, _ = get_llm_provider( model=model, @@ -5678,7 +5682,10 @@ def completion( if deployment_id is not None: # azure llms model = deployment_id custom_llm_provider = "azure" - _supplemental_provider_params: Final = {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs} + _supplemental_provider_params: Final = { + **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs}, + "extra_headers": extra_headers, + } model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, @@ -5998,6 +6005,7 @@ def completion( tpm=kwargs.get("tpm"), rpm=kwargs.get("rpm"), use_xai_oauth=kwargs.get("use_xai_oauth", False), + chatgpt_auth_file=routing_kwargs.get("chatgpt_auth_file"), gigachat_scope=kwargs.get("gigachat_scope"), gigachat_auth_url=kwargs.get("gigachat_auth_url"), gigachat_access_token=kwargs.get("gigachat_access_token"), @@ -9140,6 +9148,7 @@ async def ahealth_check( custom_llm_provider=custom_llm_provider_from_params, api_base=api_base_from_params, api_key=api_key_from_params, + litellm_params=GenericLiteLLMParams(**model_params), ) model_params["cache"] = {"no-cache": True} # don't used cached responses for making health check calls diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9a7d88b8eda..b820b058eb3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9,9 +9,8 @@ "input_cost_per_audio_token": 0.0, "input_cost_per_token": 0.0, "litellm_provider": "one of https://docs.litellm.ai/docs/providers", - "max_input_tokens": "max input tokens, if the provider specifies it. if not default to max_tokens", - "max_output_tokens": "max output tokens, if the provider specifies it. if not default to max_tokens", - "max_tokens": "LEGACY parameter. set to max_output_tokens if provider specifies it. IF not set to max_input_tokens, if provider specifies it.", + "max_input_tokens": "max input tokens, if the provider specifies it", + "max_output_tokens": "max output tokens, if the provider specifies it", "mode": "one of: chat, embedding, completion, image_generation, audio_transcription, audio_speech, image_generation, moderation, rerank, search", "output_cost_per_reasoning_token": 0.0, "output_cost_per_token": 0.0, @@ -49,37 +48,132 @@ "1024-x-1024/50-steps/stability.stable-diffusion-xl-v1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.04 }, "1024-x-1024/max-steps/stability.stable-diffusion-xl-v1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.08 }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.018 }, "512-x-512/max-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.036 }, + "abliteration/abliterated-model": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "abliteration", + "max_input_tokens": 262144, + "max_output_tokens": 262134, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://abliteration.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ] + }, + "abliteration/abliterated-model-large": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "abliteration", + "max_input_tokens": 1000000, + "max_output_tokens": 999990, + "mode": "chat", + "output_cost_per_token": 5e-06, + "source": "https://abliteration.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ] + }, + "abliteration/abliterated-model-large-v2": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "abliteration", + "max_input_tokens": 1000000, + "max_output_tokens": 999990, + "mode": "chat", + "output_cost_per_token": 5e-06, + "source": "https://abliteration.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_none_reasoning_effort": false, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ] + }, + "abliteration/abliterated-model-large-flash": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "abliteration", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://abliteration.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ] + }, "ai21.j2-mid-v1": { "input_cost_per_token": 1.25e-05, "litellm_provider": "bedrock", "max_input_tokens": 8191, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.25e-05, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -89,7 +183,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8191, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.88e-05, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -100,7 +193,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-large.md" @@ -111,7 +203,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-mini.md" @@ -121,7 +212,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 70000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -298,7 +388,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", @@ -311,7 +400,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", @@ -324,7 +412,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", @@ -337,7 +424,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", @@ -352,7 +438,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -368,7 +453,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07, @@ -389,7 +473,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -414,7 +497,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -435,7 +517,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.311e-06, "supports_function_calling": true, @@ -455,7 +536,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://aws.amazon.com/nova/pricing/", @@ -474,7 +554,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.597e-06, "supports_function_calling": true, @@ -494,7 +573,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://aws.amazon.com/nova/pricing/", @@ -518,7 +596,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-06, "output_cost_per_token_batches": 1.375e-06, @@ -544,7 +621,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -562,7 +638,6 @@ "amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 8172, - "max_tokens": 8172, "mode": "embedding", "input_cost_per_token": 1.35e-07, "input_cost_per_image": 6e-05, @@ -583,7 +658,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.4e-07, "output_cost_per_token_batches": 7e-08, @@ -602,7 +676,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.25e-05, "output_cost_per_token_batches": 6.25e-06, @@ -626,7 +699,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, "output_cost_per_token_batches": 1.6e-06, @@ -646,7 +718,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2.75e-06, @@ -660,7 +731,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -669,7 +739,6 @@ "input_cost_per_token": 8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128, - "max_tokens": 128, "metadata": { "notes": "'supports_image_input' is a deprecated field. Use 'supports_embedding_image_input' instead." }, @@ -684,7 +753,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -693,7 +761,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -702,7 +769,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -740,7 +806,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -755,7 +820,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -770,7 +834,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -784,7 +847,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 500, - "max_tokens": 500, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 512, @@ -798,7 +860,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 500, - "max_tokens": 500, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 512, @@ -812,7 +873,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 500, - "max_tokens": 500, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 512, @@ -860,7 +920,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -869,7 +928,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-07 }, @@ -878,7 +936,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -889,7 +946,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -909,7 +965,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", @@ -941,7 +996,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -964,7 +1018,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -982,7 +1035,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -1001,7 +1053,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -1021,7 +1072,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -1039,7 +1089,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -1054,7 +1103,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-06, "supports_tool_choice": true @@ -1067,7 +1115,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -1097,7 +1144,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -1125,7 +1171,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1164,7 +1209,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1204,7 +1248,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1244,7 +1287,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1284,7 +1326,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1324,7 +1365,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1364,7 +1404,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1405,7 +1444,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -1427,7 +1465,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1470,7 +1507,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1513,7 +1549,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1556,7 +1591,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1596,7 +1630,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1639,7 +1672,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1678,7 +1710,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1721,7 +1752,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1760,7 +1790,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1803,7 +1832,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1842,7 +1870,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1880,7 +1907,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1922,7 +1948,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1965,7 +1990,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "search_context_cost_per_query": { @@ -2010,7 +2034,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -2053,7 +2076,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "search_context_cost_per_query": { @@ -2098,7 +2120,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2141,7 +2162,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2186,7 +2206,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2229,7 +2248,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2269,7 +2287,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2312,7 +2329,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2357,7 +2373,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2395,7 +2410,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2435,7 +2449,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -2480,7 +2493,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -2525,7 +2537,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2570,7 +2581,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2615,7 +2625,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2660,7 +2669,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2703,7 +2711,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2745,7 +2752,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -2790,7 +2796,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -2835,7 +2840,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -2880,7 +2884,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -2925,7 +2928,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -2970,7 +2972,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -3011,7 +3012,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3050,7 +3050,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3089,7 +3088,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3128,7 +3126,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3167,7 +3164,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3206,7 +3202,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3246,7 +3241,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3286,7 +3280,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3319,7 +3312,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05 }, @@ -3328,7 +3320,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -3338,7 +3329,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-07 }, @@ -3347,7 +3337,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -3356,7 +3345,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/codellama-CodeLlama-70b-Instruct-hf" @@ -3366,7 +3354,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/google-gemma-7b-it" @@ -3376,7 +3363,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07 }, @@ -3385,7 +3371,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -3394,7 +3379,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07 }, @@ -3403,7 +3387,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/meta-llama-Meta-Llama-3-70B-Instruct" @@ -3413,7 +3396,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/meta-llama-Meta-Llama-3-8B-Instruct" @@ -3423,7 +3405,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mistral-7B-Instruct-v0.1", @@ -3434,7 +3415,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mixtral-8x22B-Instruct-v0.1", @@ -3445,7 +3425,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mixtral-8x7B-Instruct-v0.1", @@ -3457,7 +3436,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.52e-07, "supports_function_calling": true, @@ -3473,7 +3451,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.48e-07, "supports_function_calling": true, @@ -3489,7 +3466,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.36e-06, "output_cost_per_token_flex": 1.68e-06, @@ -3508,7 +3484,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -3543,7 +3518,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3595,7 +3569,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3628,7 +3601,6 @@ "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -3640,7 +3612,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 6e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -3670,7 +3641,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -3685,7 +3655,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -3709,7 +3678,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, @@ -3734,7 +3702,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3765,7 +3732,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3797,7 +3763,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3830,7 +3795,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3868,7 +3832,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3899,7 +3862,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3938,7 +3900,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3971,7 +3932,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -3996,7 +3956,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -4030,7 +3989,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": false, @@ -4052,7 +4010,6 @@ "litellm_provider": "azure", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -4087,7 +4044,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, @@ -4105,7 +4061,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -4154,7 +4109,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -4203,7 +4157,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -4252,7 +4205,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -4304,7 +4256,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -4345,7 +4296,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", @@ -4395,7 +4345,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 6e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", @@ -4439,7 +4388,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -4486,7 +4434,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -4533,7 +4480,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -4581,7 +4527,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -4629,7 +4574,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -4675,7 +4619,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -4717,7 +4660,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_priority": 9e-06, @@ -4759,7 +4701,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_priority": 9e-06, @@ -4801,7 +4742,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_priority": 2.5e-06, @@ -4843,7 +4783,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_priority": 2.5e-06, @@ -4883,7 +4822,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" @@ -4895,7 +4833,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Catalog-name twin of azure_ai/model_router: the flat $0.14 per M input tokens is the router's own fee, the routed model is priced on top of it" @@ -4908,7 +4845,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -4929,7 +4865,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -4949,7 +4884,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -4968,7 +4902,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -4987,7 +4920,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5006,7 +4938,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2.2e-05, "output_cost_per_token": 2.64e-06, @@ -5026,7 +4957,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 0.00022, "output_cost_per_token": 2.2e-05, @@ -5046,7 +4976,6 @@ "litellm_provider": "azure", "max_input_tokens": 16000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 8.8e-05, "output_cost_per_token": 2.2e-05, @@ -5076,7 +5005,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -5116,7 +5044,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "output_cost_per_token_batches": 1.1e-06, @@ -5155,7 +5082,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "supported_endpoints": [ @@ -5196,7 +5122,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5228,7 +5153,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5261,7 +5185,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, @@ -5298,7 +5221,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -5317,7 +5239,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -5335,7 +5256,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -5353,7 +5273,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -5371,7 +5290,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -5389,7 +5307,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5409,7 +5326,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5426,7 +5342,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -5443,7 +5358,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5463,7 +5377,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5484,7 +5397,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -5525,7 +5437,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5557,7 +5468,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5587,7 +5497,6 @@ "litellm_provider": "azure", "max_input_tokens": 4097, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -5597,7 +5506,7 @@ "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", "max_input_tokens": 4097, - "max_tokens": 4097, + "max_output_tokens": 4097, "mode": "completion", "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -5607,7 +5516,6 @@ "litellm_provider": "azure", "max_input_tokens": 4097, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -5619,7 +5527,6 @@ "litellm_provider": "azure", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5631,7 +5538,6 @@ "litellm_provider": "azure", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5642,7 +5548,7 @@ "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", "max_input_tokens": 4097, - "max_tokens": 4097, + "max_output_tokens": 4097, "mode": "completion", "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -5651,7 +5557,7 @@ "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", "max_input_tokens": 4097, - "max_tokens": 4097, + "max_output_tokens": 4097, "mode": "completion", "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -5662,7 +5568,6 @@ "litellm_provider": "azure", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -5674,7 +5579,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -5687,7 +5591,6 @@ "litellm_provider": "azure", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5699,7 +5602,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -5712,7 +5614,6 @@ "litellm_provider": "azure", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.00012, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5724,7 +5625,6 @@ "litellm_provider": "azure", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.00012, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5735,7 +5635,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/", @@ -5748,7 +5647,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/", @@ -5762,7 +5660,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/", @@ -5779,7 +5676,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -5817,7 +5713,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -5855,7 +5750,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, @@ -5893,7 +5787,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, @@ -5929,7 +5822,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -5963,7 +5855,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -5997,7 +5888,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 0.00015, "output_cost_per_token_batches": 7.5e-05, @@ -6017,7 +5907,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -6034,7 +5923,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -6053,7 +5941,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -6073,7 +5960,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -6092,7 +5978,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -6125,7 +6010,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -6157,7 +6041,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -6190,7 +6073,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -6206,6 +6088,8 @@ "text", "audio" ], + "supports_audio_input": true, + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6223,7 +6107,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6255,7 +6138,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6270,6 +6152,8 @@ "text", "audio" ], + "supports_audio_input": true, + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6286,7 +6170,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -6318,7 +6201,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -6336,7 +6218,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -6354,7 +6235,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 6e-07, @@ -6388,7 +6268,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6410,7 +6289,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6445,7 +6323,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6479,7 +6356,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6514,7 +6390,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6548,7 +6423,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -6583,7 +6457,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6618,7 +6491,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6652,7 +6524,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6716,7 +6587,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 0.0002, "output_cost_per_token": 2e-05, @@ -6735,7 +6605,6 @@ "litellm_provider": "azure", "max_input_tokens": 16000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 2e-05, @@ -6789,7 +6658,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "audio_transcription", "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ @@ -6813,7 +6681,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -6856,7 +6723,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -6891,7 +6757,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "output_cost_per_token_priority": 3.6e-06, @@ -6928,7 +6793,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -6967,7 +6831,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -7004,7 +6867,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7040,7 +6902,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -7079,7 +6940,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -7118,7 +6978,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -7154,7 +7013,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -7190,7 +7048,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -7224,7 +7081,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -7265,7 +7121,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7290,7 +7145,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7312,7 +7166,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7333,7 +7186,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7354,7 +7206,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7376,7 +7227,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7397,7 +7247,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7418,7 +7267,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7440,7 +7288,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7461,7 +7308,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7482,7 +7328,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.6e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7498,7 +7343,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.6e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7514,7 +7358,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7546,7 +7389,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7578,7 +7420,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7614,7 +7455,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_batches": 7e-06, @@ -7654,7 +7494,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_batches": 7e-06, @@ -7692,7 +7531,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7726,7 +7564,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -7758,7 +7595,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -7797,7 +7633,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -7846,7 +7681,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -7902,7 +7736,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -7952,7 +7785,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -8002,7 +7834,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -8058,7 +7889,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -8108,7 +7938,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -8159,7 +7988,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -8208,7 +8036,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -8262,7 +8089,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -8323,7 +8149,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -8387,7 +8212,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -8451,7 +8275,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -8515,7 +8338,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -8579,7 +8401,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, @@ -8643,7 +8464,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, @@ -8695,7 +8515,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -8744,7 +8563,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -8793,7 +8611,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -8842,7 +8659,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -8897,7 +8713,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -8950,7 +8765,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -8997,7 +8811,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -9046,7 +8859,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -9095,7 +8907,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -9144,7 +8955,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -9193,7 +9003,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -9238,7 +9047,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "reasoning_effort_levels": [ @@ -9275,7 +9083,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "reasoning_effort_levels": [ @@ -9321,7 +9128,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9376,7 +9182,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9432,7 +9237,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, @@ -9488,7 +9292,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, @@ -9538,7 +9341,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "output_cost_per_token_above_272k_tokens": 8.25e-05, @@ -9587,7 +9389,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "output_cost_per_token_above_272k_tokens": 8.25e-07, @@ -9642,7 +9443,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -9685,7 +9485,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.3e-05, "reasoning_effort_levels": [ @@ -9731,7 +9530,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9786,7 +9584,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9842,7 +9639,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, @@ -9898,7 +9694,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, @@ -9957,7 +9752,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -10016,7 +9810,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, @@ -10071,7 +9864,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, @@ -10123,7 +9915,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -10183,7 +9974,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_batches": 1.5e-05, @@ -10242,7 +10032,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10295,7 +10084,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10349,7 +10137,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10402,7 +10189,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10449,7 +10235,6 @@ "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -10493,7 +10278,6 @@ "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -10535,7 +10319,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "search_context_cost_per_query": { @@ -10586,7 +10369,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "search_context_cost_per_query": { @@ -10638,7 +10420,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "search_context_cost_per_query": { @@ -10686,7 +10467,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "search_context_cost_per_query": { @@ -11062,7 +10842,7 @@ "input_cost_per_token": 8e-06, "litellm_provider": "azure", "max_input_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_function_calling": true @@ -11071,7 +10851,7 @@ "input_cost_per_token": 8e-06, "litellm_provider": "azure", "max_input_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_function_calling": true @@ -11083,7 +10863,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_function_calling": true, @@ -11101,7 +10880,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -11120,7 +10898,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11138,7 +10915,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11156,7 +10932,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "supported_endpoints": [ @@ -11187,7 +10962,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -11219,7 +10993,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, "search_context_cost_per_query": { @@ -11257,7 +11030,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_prompt_caching": true, @@ -11274,7 +11046,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11291,7 +11062,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -11322,7 +11092,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -11353,7 +11122,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supported_endpoints": [ @@ -11384,7 +11152,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11429,7 +11196,6 @@ "input_cost_per_token": 1.43e-07, "litellm_provider": "azure", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -11439,7 +11205,6 @@ "input_cost_per_token": 2.2e-08, "litellm_provider": "azure", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -11449,7 +11214,6 @@ "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -11500,7 +11264,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -11538,7 +11301,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.76e-06, "output_cost_per_token_batches": 8.8e-07, @@ -11574,7 +11336,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, @@ -11608,7 +11369,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11629,7 +11389,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11649,7 +11408,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11668,7 +11426,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11687,7 +11444,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11706,7 +11462,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2.2e-05, "output_cost_per_token": 2.64e-06, @@ -11726,7 +11481,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 0.00022, "output_cost_per_token": 2.2e-05, @@ -11746,7 +11500,6 @@ "litellm_provider": "azure", "max_input_tokens": 16000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 8.8e-05, "output_cost_per_token": 2.2e-05, @@ -11776,7 +11529,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11816,7 +11568,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "output_cost_per_token_batches": 1.1e-06, @@ -11854,7 +11605,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, @@ -11892,7 +11642,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "supported_endpoints": [ @@ -11933,7 +11682,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11965,7 +11713,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11998,7 +11745,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -12017,7 +11763,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -12035,7 +11780,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12053,7 +11797,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -12086,7 +11829,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -12119,7 +11861,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12137,7 +11878,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12155,7 +11895,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12176,7 +11915,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12200,7 +11938,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -12211,7 +11948,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -12223,7 +11959,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12241,7 +11976,6 @@ "azure_ai/FLUX-1.1-pro": { "litellm_provider": "azure_ai", "max_input_tokens": 5000, - "max_tokens": 5000, "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659", @@ -12252,7 +11986,6 @@ "azure_ai/FLUX.1-Kontext-pro": { "litellm_provider": "azure_ai", "max_input_tokens": 5000, - "max_tokens": 5000, "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/black-forest-labs/", @@ -12263,7 +11996,6 @@ "azure_ai/flux.2-pro": { "litellm_provider": "azure_ai", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -12275,7 +12007,6 @@ "input_cost_per_pixel": 5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "image_generation", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/black-forest-labs/", "supported_endpoints": [ @@ -12297,7 +12028,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.85e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12312,7 +12042,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.828e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12328,7 +12057,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.52e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12344,7 +12072,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12359,7 +12086,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12374,7 +12100,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 7.26e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12389,7 +12114,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.46e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12411,7 +12135,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.3e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12434,7 +12157,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12457,7 +12179,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12480,7 +12201,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ @@ -12509,7 +12229,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12524,7 +12243,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12567,7 +12285,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.64e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12633,7 +12350,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12656,7 +12372,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.1e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -12668,7 +12383,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12681,7 +12395,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 10000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/", @@ -12695,7 +12408,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 8192, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12707,7 +12419,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.54e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12719,7 +12430,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12732,7 +12442,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12745,7 +12454,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12758,7 +12466,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12771,7 +12478,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12784,7 +12490,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12797,7 +12502,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12810,7 +12514,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12823,7 +12526,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12836,7 +12538,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12850,7 +12551,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12863,7 +12563,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.2e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12877,7 +12576,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12889,7 +12587,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12931,7 +12628,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 8182, - "max_tokens": 8182, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12971,7 +12667,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12985,7 +12680,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" @@ -12997,7 +12691,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" @@ -13008,7 +12701,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076" @@ -13019,7 +12711,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076" @@ -13029,7 +12720,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13044,7 +12734,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13060,7 +12749,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.56e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13072,7 +12760,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13088,7 +12775,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 5.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13105,7 +12791,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13118,7 +12803,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 3072, @@ -13138,7 +12822,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13155,7 +12838,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -13176,7 +12858,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -13196,7 +12877,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13215,7 +12895,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13232,7 +12911,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, @@ -13246,7 +12924,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, @@ -13260,7 +12937,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13275,7 +12951,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.00971, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" @@ -13286,7 +12961,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 70000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13298,7 +12972,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13315,7 +12988,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13339,7 +13011,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-08, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", @@ -13352,7 +13023,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13365,7 +13035,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13377,7 +13046,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", @@ -13389,7 +13057,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13403,7 +13070,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "supports_assistant_prefill": true, @@ -13416,7 +13082,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13428,7 +13093,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13441,7 +13105,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -13455,7 +13118,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -13467,7 +13129,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13477,7 +13138,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13487,7 +13147,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13497,7 +13156,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13522,7 +13180,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13532,7 +13189,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v2:1": { @@ -13541,7 +13197,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13551,7 +13206,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13561,7 +13215,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v2:1": { @@ -13570,7 +13223,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13579,7 +13231,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 7.55e-06, "supports_tool_choice": true @@ -13589,7 +13240,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -13599,7 +13249,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -13610,7 +13259,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -13630,7 +13278,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13648,7 +13295,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13668,7 +13314,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, "output_cost_per_token_flex": 1.52e-06, @@ -13682,7 +13327,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -13700,7 +13344,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13719,7 +13362,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -13733,7 +13375,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -13753,7 +13394,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -13764,7 +13404,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -13775,7 +13414,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -13795,7 +13433,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13813,7 +13450,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13833,7 +13469,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.94e-06, "output_cost_per_token_flex": 1.47e-06, @@ -13847,7 +13482,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -13865,7 +13499,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13883,7 +13516,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13902,7 +13534,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -13922,7 +13553,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13940,7 +13570,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13959,7 +13588,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -13977,7 +13605,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13995,7 +13622,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.03e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14006,7 +13632,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.9e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14017,7 +13642,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -14037,7 +13661,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14055,7 +13678,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14074,7 +13696,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -14092,7 +13713,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14102,7 +13722,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-v2:1": { @@ -14111,7 +13730,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14121,7 +13739,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14131,7 +13748,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/eu-central-1/6-month-commitment/anthropic.claude-v2:1": { @@ -14140,7 +13756,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14149,7 +13764,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 8.38e-06, "supports_tool_choice": true @@ -14159,7 +13773,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05 }, @@ -14168,7 +13781,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -14180,7 +13792,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14198,7 +13809,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14217,7 +13827,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14235,7 +13844,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.78e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14246,7 +13854,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14258,7 +13865,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14276,7 +13882,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14295,7 +13900,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14313,7 +13917,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.55e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14324,7 +13927,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14336,7 +13938,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, "output_cost_per_token_flex": 9.3e-07, @@ -14354,7 +13955,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14373,7 +13973,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.01e-06, "output_cost_per_token_flex": 5.05e-07, @@ -14392,7 +13991,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.86e-06, "output_cost_per_token_flex": 9.3e-07, @@ -14410,7 +14008,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -14421,7 +14018,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.56e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -14433,7 +14029,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9.1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -14446,7 +14041,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14464,7 +14058,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14483,7 +14076,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14500,7 +14092,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Anthropic via Invoke route does not currently support pdf input." }, @@ -14519,7 +14110,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.88e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14530,7 +14120,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.01e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14541,7 +14130,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -14561,7 +14149,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14579,7 +14166,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14599,7 +14185,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, "output_cost_per_token_flex": 1.52e-06, @@ -14613,7 +14198,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -14631,7 +14215,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14649,7 +14232,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14659,7 +14241,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-v2:1": { @@ -14668,7 +14249,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14678,7 +14258,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14688,7 +14267,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-east-1/6-month-commitment/anthropic.claude-v2:1": { @@ -14697,7 +14275,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14706,7 +14283,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-06, "supports_tool_choice": true @@ -14716,7 +14292,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -14726,7 +14301,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -14737,7 +14311,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14748,7 +14321,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14759,7 +14331,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -14770,7 +14341,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -14782,7 +14352,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -14794,7 +14363,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_flex": 9.25e-07, @@ -14814,7 +14382,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -14834,7 +14401,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -14851,7 +14417,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -14865,7 +14430,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -14883,7 +14447,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -14901,7 +14464,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_flex": 9.25e-07, @@ -14921,7 +14483,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -14939,7 +14500,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14959,7 +14519,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -14973,7 +14532,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -14991,7 +14549,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -15009,7 +14566,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.84e-06, "supports_function_calling": true, @@ -15023,7 +14579,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -15032,7 +14587,6 @@ "input_cost_per_token": 2e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -15042,7 +14596,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.7e-06 }, @@ -15051,7 +14604,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-07 }, @@ -15060,7 +14612,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -15073,7 +14624,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15102,7 +14652,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15123,7 +14672,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.5e-06, "supports_pdf_input": true @@ -15133,7 +14681,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2.65e-06, "supports_pdf_input": true @@ -15141,7 +14688,6 @@ "bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 8172, - "max_tokens": 8172, "mode": "embedding", "input_cost_per_token": 1.62e-07, "input_cost_per_image": 7.2e-05, @@ -15160,7 +14706,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.88e-07, "supports_function_calling": true, @@ -15176,7 +14721,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.68e-07, "supports_function_calling": true, @@ -15190,7 +14734,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.84e-06, "supports_function_calling": true, @@ -15204,7 +14747,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -15213,7 +14755,6 @@ "input_cost_per_token": 1.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -15223,7 +14764,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.6e-06 }, @@ -15232,7 +14772,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-07 }, @@ -15241,7 +14780,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -15254,7 +14792,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15283,7 +14820,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15304,7 +14840,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.5e-06, "supports_pdf_input": true @@ -15314,7 +14849,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6e-07, "supports_pdf_input": true @@ -15325,7 +14859,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -15336,7 +14869,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -15347,7 +14879,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15357,7 +14888,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-v2:1": { @@ -15366,7 +14896,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15376,7 +14905,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15386,7 +14914,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-west-2/6-month-commitment/anthropic.claude-v2:1": { @@ -15395,7 +14922,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15404,7 +14930,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-06, "supports_tool_choice": true @@ -15414,7 +14939,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -15424,7 +14948,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -15435,7 +14958,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -15446,7 +14968,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -15458,7 +14979,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -15470,7 +14990,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_flex": 9.25e-07, @@ -15490,7 +15009,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -15510,7 +15028,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -15527,7 +15044,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -15541,7 +15057,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -15559,7 +15074,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -15578,7 +15092,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -15668,7 +15181,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -15679,7 +15191,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -15690,7 +15201,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true, @@ -15701,7 +15211,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -15712,7 +15221,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -15723,7 +15231,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.5e-07, "source": "https://www.cerebras.ai/blog/openai-gpt-oss-120b-runs-fastest-on-cerebras", @@ -15738,7 +15245,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://inference-docs.cerebras.ai/support/pricing", @@ -15751,7 +15257,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.49e-06, "source": "https://api.cerebras.ai/public/v1/models/qwen-3.8-27b", @@ -15767,7 +15272,6 @@ "litellm_provider": "nlp_cloud", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5e-07 }, @@ -15798,7 +15302,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -15825,7 +15328,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -15854,7 +15356,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -15888,7 +15389,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -15924,7 +15424,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -15968,7 +15467,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -16015,7 +15513,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -16059,7 +15556,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -16088,7 +15584,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16122,7 +15617,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16159,7 +15653,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16202,7 +15695,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16245,7 +15737,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16289,7 +15780,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16333,7 +15823,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -16378,7 +15867,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -16424,7 +15912,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1e-05, @@ -16470,7 +15957,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16518,7 +16004,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16634,7 +16119,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 3072, "max_output_tokens": 3072, - "max_tokens": 3072, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16644,7 +16128,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 2048, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16654,7 +16137,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16664,7 +16146,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16674,7 +16155,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-07, "rpm": 300, @@ -16686,7 +16166,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16696,7 +16175,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 80000, "max_output_tokens": 80000, - "max_tokens": 80000, "mode": "chat", "output_cost_per_token": 3.35e-07, "rpm": 300 @@ -16706,7 +16184,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-08, "rpm": 300 @@ -16716,7 +16193,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 15000, "max_output_tokens": 15000, - "max_tokens": 15000, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16727,7 +16203,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "rpm": 20, @@ -16739,7 +16214,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 80000, "max_output_tokens": 80000, - "max_tokens": 80000, "mode": "chat", "output_cost_per_token": 4.881e-06, "rpm": 300, @@ -16750,7 +16224,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.87e-07, "rpm": 300 @@ -16760,7 +16233,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 60000, "max_output_tokens": 60000, - "max_tokens": 60000, "mode": "chat", "output_cost_per_token": 2.01e-07, "rpm": 300 @@ -16771,7 +16243,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "rpm": 20, @@ -16783,7 +16254,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-07, "rpm": 300, @@ -16795,7 +16265,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16805,7 +16274,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 24000, "max_output_tokens": 24000, - "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 2.253e-06, "rpm": 300, @@ -16816,7 +16284,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.12e-07, "rpm": 300, @@ -16827,7 +16294,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1e-06, "rpm": 300 @@ -16838,7 +16304,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "rpm": 20, @@ -16850,7 +16315,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-06, "rpm": 300, @@ -16862,7 +16326,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.55e-07, "rpm": 300 @@ -16872,7 +16335,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.35e-07, "rpm": 300, @@ -16884,7 +16346,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 3500, "max_output_tokens": 3500, - "max_tokens": 3500, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16894,7 +16355,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, "rpm": 300, @@ -16906,7 +16366,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.55e-07, "rpm": 300, @@ -16917,7 +16376,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.76e-07, "rpm": 300, @@ -16928,7 +16386,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "rpm": 300, @@ -16940,7 +16397,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 8.5e-07, "rpm": 300, @@ -16951,7 +16407,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 24000, "max_output_tokens": 24000, - "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 1e-06, "rpm": 300, @@ -16962,7 +16417,6 @@ "litellm_provider": "codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/", @@ -16974,7 +16428,6 @@ "litellm_provider": "codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/", @@ -16986,7 +16439,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -16997,7 +16449,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17007,7 +16458,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17017,7 +16467,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17027,7 +16476,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17038,7 +16486,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17049,7 +16496,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17060,7 +16506,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "cohere", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17072,7 +16517,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "cohere", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048, @@ -17083,7 +16527,6 @@ "input_cost_per_token": 8e-08, "litellm_provider": "cohere", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048, @@ -17105,7 +16548,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-cohere-rerank-3-5.md" @@ -17115,7 +16557,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -17127,7 +16568,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "source": "https://docs.cohere.com/docs/models" }, @@ -17136,7 +16576,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.cohere.com/docs/command-a-plus", @@ -17151,7 +16590,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06 }, @@ -17160,7 +16598,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -17171,7 +16608,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -17182,7 +16618,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.cohere.com/v2/docs/command-r7b", @@ -17195,7 +16630,7 @@ "input_cost_per_token": 1.12e-06, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17218,7 +16653,7 @@ "input_cost_per_token": 1.12e-06, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17241,7 +16676,7 @@ "input_cost_per_token": 1.12e-06, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17264,7 +16699,7 @@ "input_cost_per_token": 1e-08, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17287,7 +16722,7 @@ "input_cost_per_token": 1e-08, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17310,7 +16745,7 @@ "input_cost_per_token": 1e-08, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17333,7 +16768,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, "supported_endpoints": [ @@ -17363,7 +16797,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, "supported_endpoints": [ @@ -17394,7 +16827,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -17416,7 +16848,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -17438,7 +16869,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17454,7 +16884,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17470,7 +16899,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4.8e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17486,7 +16914,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 202745, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17502,7 +16929,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17518,7 +16944,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 229376, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17534,7 +16959,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17546,7 +16970,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17575,7 +16998,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17605,7 +17027,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17618,7 +17039,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17631,7 +17051,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17644,7 +17063,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -17658,7 +17076,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -17671,7 +17088,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17702,7 +17118,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17733,7 +17148,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17765,7 +17179,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -17779,7 +17192,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17792,7 +17204,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -17806,7 +17217,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -17819,7 +17229,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17830,7 +17239,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17879,7 +17287,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17924,7 +17331,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17973,7 +17379,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18018,7 +17423,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18055,7 +17459,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18092,7 +17495,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18130,7 +17532,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18142,7 +17543,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18155,7 +17555,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18168,7 +17567,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18182,7 +17580,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18195,7 +17592,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.87e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18208,7 +17604,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 260096, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18246,7 +17641,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18280,7 +17674,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -18294,7 +17687,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18330,7 +17722,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -18348,7 +17739,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -18367,7 +17757,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -18386,7 +17775,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -18707,7 +18095,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.qwencloud.com/models", @@ -18723,7 +18110,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.qwencloud.com/models", @@ -18739,7 +18125,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4.8e-06, "source": "https://www.qwencloud.com/models", @@ -18755,7 +18140,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 202745, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.qwencloud.com/models", @@ -18771,7 +18155,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.qwencloud.com/models", @@ -18787,7 +18170,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 229376, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.qwencloud.com/models", @@ -18803,7 +18185,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.qwencloud.com/models", @@ -18815,7 +18196,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -18844,7 +18224,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -18874,7 +18253,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.qwencloud.com/models", @@ -18887,7 +18265,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -18900,7 +18277,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -18913,7 +18289,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -18927,7 +18302,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -18940,7 +18314,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -18971,7 +18344,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19002,7 +18374,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19034,7 +18405,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -19048,7 +18418,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.qwencloud.com/models", @@ -19061,7 +18430,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -19075,7 +18443,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -19088,7 +18455,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19099,7 +18465,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19148,7 +18513,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19193,7 +18557,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19242,7 +18605,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19287,7 +18649,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19324,7 +18685,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19361,7 +18721,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19399,7 +18758,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -19411,7 +18769,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -19424,7 +18781,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://www.qwencloud.com/models", @@ -19437,7 +18793,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.qwencloud.com/models", @@ -19451,7 +18806,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://www.qwencloud.com/models", @@ -19464,7 +18818,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.87e-06, "source": "https://www.qwencloud.com/models", @@ -19477,7 +18830,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 260096, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19515,7 +18867,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19549,7 +18900,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://www.qwencloud.com/models", @@ -19563,7 +18913,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19599,7 +18948,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://www.qwencloud.com/models", @@ -19615,7 +18963,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.qwencloud.com/models", @@ -19936,7 +19283,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -19952,7 +19298,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -19968,7 +19313,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4.8e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -19984,7 +19328,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 202745, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20000,7 +19343,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20016,7 +19358,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 229376, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20032,7 +19373,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20044,7 +19384,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20073,7 +19412,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20103,7 +19441,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20116,7 +19453,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20129,7 +19465,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20142,7 +19477,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -20156,7 +19490,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -20169,7 +19502,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20200,7 +19532,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20231,7 +19562,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20263,7 +19593,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -20277,7 +19606,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20290,7 +19618,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -20304,7 +19631,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -20317,7 +19643,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20328,7 +19653,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20377,7 +19701,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20422,7 +19745,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20471,7 +19793,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20516,7 +19837,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20553,7 +19873,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20590,7 +19909,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20628,7 +19946,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20640,7 +19957,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20653,7 +19969,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20666,7 +19981,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20680,7 +19994,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20693,7 +20006,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.87e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20706,7 +20018,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 260096, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20744,7 +20055,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20778,7 +20088,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20792,7 +20101,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20828,7 +20136,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20846,7 +20153,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -20865,7 +20171,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -20884,7 +20189,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -21206,7 +20510,6 @@ "input_dbu_cost_per_token": 1.429e-06, "litellm_provider": "databricks", "max_input_tokens": 512, - "max_tokens": 512, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21224,7 +20527,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21253,7 +20555,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21283,7 +20584,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21307,7 +20607,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21331,7 +20630,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21355,7 +20653,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21380,7 +20677,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21405,7 +20701,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21432,7 +20727,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21460,7 +20754,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21489,7 +20782,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21524,7 +20816,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21548,7 +20839,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21571,7 +20861,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21595,7 +20884,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21620,7 +20908,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields. Introductory launch rates of 28.571 input / 142.857 output / 35.714 cache write / 2.857 cache read DBU run through 2026-08-31; the standard rates are listed here because entries carry no expiry date." }, @@ -21649,7 +20936,6 @@ "token_count_framing": 97, "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference. Context/max output are the DeepSeek-published model limits (1M context, 384K max output)." }, @@ -21682,7 +20968,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference. Context/max output are the DeepSeek-published model limits (1M context, 384K max output)." }, @@ -21711,7 +20996,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21745,7 +21029,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21780,7 +21063,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21808,7 +21090,6 @@ "litellm_provider": "databricks", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21829,7 +21110,6 @@ "litellm_provider": "databricks", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21854,7 +21134,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21887,7 +21166,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21919,7 +21197,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21935,7 +21212,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21960,7 +21236,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21989,7 +21264,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22021,7 +21295,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22052,7 +21325,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22085,7 +21357,6 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22102,7 +21373,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference." }, @@ -22130,7 +21400,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22161,7 +21430,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22191,7 +21459,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22211,7 +21478,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22231,7 +21497,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22251,7 +21516,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22271,7 +21535,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22300,7 +21563,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22329,7 +21591,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22358,7 +21619,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields. Rates reflect OpenAI's promotional pricing in effect through November 21, 2026; afterwards input, cache and Batch rates are 25% higher and output rates 50% higher." }, @@ -22387,7 +21647,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22416,7 +21675,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22445,7 +21703,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22474,7 +21731,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22503,7 +21759,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22523,7 +21778,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22543,7 +21797,6 @@ "litellm_provider": "databricks", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22560,7 +21813,6 @@ "litellm_provider": "databricks", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22602,7 +21854,6 @@ "input_dbu_cost_per_token": 1.857e-06, "litellm_provider": "databricks", "max_input_tokens": 8192, - "max_tokens": 8192, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22649,7 +21900,6 @@ "token_count_framing": 100, "max_input_tokens": 1000000, "max_output_tokens": 1048576, - "max_tokens": 1048576, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference." }, @@ -22683,7 +21933,6 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)." }, @@ -22701,7 +21950,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22718,7 +21966,6 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22736,7 +21983,6 @@ "litellm_provider": "databricks", "max_input_tokens": 262144, "max_output_tokens": 25000, - "max_tokens": 25000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22786,7 +22032,6 @@ "input_dbu_cost_per_token": 2.86e-07, "litellm_provider": "databricks", "max_input_tokens": 32768, - "max_tokens": 32768, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22832,7 +22077,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -23427,7 +22671,6 @@ ] }, "deepinfra/Gryphe/MythoMax-L2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 4e-07, @@ -23439,7 +22682,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/NousResearch/Hermes-3-Llama-3.1-405B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-06, @@ -23450,7 +22692,6 @@ "supports_function_calling": true }, "deepinfra/NousResearch/Hermes-3-Llama-3.1-70B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 7e-07, @@ -23461,7 +22702,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/QwQ-32B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.5e-07, @@ -23472,7 +22712,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen2.5-72B-Instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 3.6e-07, @@ -23484,7 +22723,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen2.5-7B-Instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 4e-08, @@ -23494,7 +22732,6 @@ "supports_tool_choice": false }, "deepinfra/Qwen/Qwen2.5-VL-32B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -23506,7 +22743,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-14B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1.2e-07, @@ -23518,7 +22754,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-235B-A22B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1.8e-07, @@ -23529,7 +22764,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-08, @@ -23541,7 +22775,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 3e-07, @@ -23552,7 +22785,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-30B-A3B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1.2e-07, @@ -23564,7 +22796,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-32B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 8e-08, @@ -23576,7 +22807,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 4e-07, @@ -23587,7 +22817,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct-Turbo": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 3e-07, @@ -23601,7 +22830,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Next-80B-A3B-Instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-08, @@ -23613,7 +22841,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Next-80B-A3B-Thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.4e-07, @@ -23624,7 +22851,6 @@ "supports_function_calling": true }, "deepinfra/Sao10K/L3-8B-Lunaris-v1-Turbo": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 4e-08, @@ -23634,7 +22860,6 @@ "supports_tool_choice": false }, "deepinfra/Sao10K/L3.1-70B-Euryale-v2.2": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 8.5e-07, @@ -23645,7 +22870,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Sao10K/L3.3-70B-Euryale-v2.3": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6.5e-07, @@ -23655,7 +22879,6 @@ "supports_tool_choice": false }, "deepinfra/allenai/olmOCR-7B-0725-FP8": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2.7e-07, @@ -23665,7 +22888,6 @@ "supports_tool_choice": false }, "deepinfra/anthropic/claude-3-7-sonnet-latest": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 3.3e-06, @@ -23677,7 +22899,6 @@ "supports_function_calling": true }, "deepinfra/anthropic/claude-4-opus": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 1.65e-05, @@ -23688,7 +22909,6 @@ "supports_function_calling": true }, "deepinfra/anthropic/claude-4-sonnet": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 3.3e-06, @@ -23699,7 +22919,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 7e-07, @@ -23710,7 +22929,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-0528": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -23722,7 +22940,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-0528-Turbo": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-06, @@ -23733,7 +22950,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -23743,7 +22959,6 @@ "supports_tool_choice": false }, "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.7e-07, @@ -23754,7 +22969,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-Turbo": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-06, @@ -23765,7 +22979,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-V3": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 3.2e-07, @@ -23777,7 +22990,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3-0324": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 2.4e-07, @@ -23791,7 +23003,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.1": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 2.5e-07, @@ -23805,7 +23016,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.1-Terminus": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 2.7e-07, @@ -23817,7 +23027,6 @@ "supports_function_calling": true }, "deepinfra/google/gemini-2.5-flash": { - "max_tokens": 1000000, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "input_cost_per_token": 3e-07, @@ -23833,7 +23042,6 @@ "supports_audio_input": true }, "deepinfra/google/gemini-2.5-pro": { - "max_tokens": 1000000, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "input_cost_per_token": 1.25e-06, @@ -23848,7 +23056,6 @@ "supports_audio_input": true }, "deepinfra/google/gemma-3-12b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-08, @@ -23860,7 +23067,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-3-27b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 8e-08, @@ -23872,7 +23078,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-3-4b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-08, @@ -23884,7 +23089,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-3.2-11B-Vision-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4.9e-08, @@ -23894,7 +23098,6 @@ "supports_tool_choice": false }, "deepinfra/meta-llama/Llama-3.2-3B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, @@ -23905,7 +23108,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.3e-07, @@ -23916,7 +23118,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -23928,7 +23129,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 2e-07, @@ -23940,7 +23140,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "max_tokens": 327680, "max_input_tokens": 327680, "max_output_tokens": 327680, "input_cost_per_token": 1e-07, @@ -23952,7 +23151,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-Guard-3-8B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5.5e-08, @@ -23962,7 +23160,6 @@ "supports_tool_choice": false }, "deepinfra/meta-llama/Llama-Guard-4-12B": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1.8e-07, @@ -23972,7 +23169,6 @@ "supports_tool_choice": false }, "deepinfra/meta-llama/Meta-Llama-3-8B-Instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 3e-08, @@ -23983,7 +23179,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-07, @@ -23994,7 +23189,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-07, @@ -24006,7 +23200,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -24017,7 +23210,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, @@ -24029,7 +23221,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/microsoft/WizardLM-2-8x22B": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 4.8e-07, @@ -24039,7 +23230,6 @@ "supports_tool_choice": false }, "deepinfra/microsoft/phi-4": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 7e-08, @@ -24050,7 +23240,6 @@ "supports_function_calling": true }, "deepinfra/mistralai/Mistral-Nemo-Instruct-2407": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.9e-08, @@ -24062,7 +23251,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/mistralai/Mistral-Small-24B-Instruct-2501": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-08, @@ -24073,7 +23261,6 @@ "supports_function_calling": true }, "deepinfra/mistralai/Mistral-Small-3.2-24B-Instruct-2506": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 7.5e-08, @@ -24084,7 +23271,6 @@ "supports_function_calling": true }, "deepinfra/mistralai/Mixtral-8x7B-Instruct-v0.1": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 4e-07, @@ -24095,7 +23281,6 @@ "supports_function_calling": true }, "deepinfra/moonshotai/Kimi-K2-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-07, @@ -24106,7 +23291,6 @@ "supports_function_calling": true }, "deepinfra/moonshotai/Kimi-K2-Instruct-0905": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 5e-07, @@ -24118,7 +23302,6 @@ "supports_function_calling": true }, "deepinfra/nvidia/Llama-3.1-Nemotron-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6e-07, @@ -24129,7 +23312,6 @@ "supports_function_calling": true }, "deepinfra/nvidia/Llama-3.3-Nemotron-Super-49B-v1.5": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -24153,7 +23335,6 @@ "supports_prompt_caching": true }, "deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-08, @@ -24164,7 +23345,6 @@ "supports_function_calling": true }, "deepinfra/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3.7e-08, @@ -24176,7 +23356,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/openai/gpt-oss-20b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -24188,7 +23367,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.5": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-07, @@ -24207,7 +23385,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -24230,7 +23407,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07, "supports_assistant_prefill": true, @@ -24245,7 +23421,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.19e-06, "supports_assistant_prefill": true, @@ -24262,7 +23437,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -24287,7 +23461,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.1e-06, "supports_assistant_prefill": true, @@ -24302,7 +23475,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 4e-07, "supports_assistant_prefill": true, @@ -24317,7 +23489,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-1.html", @@ -24334,7 +23505,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 164000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_batches": 9.25e-07, @@ -24354,7 +23524,6 @@ "litellm_provider": "nlp_cloud", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "completion", "output_cost_per_token": 5e-07 }, @@ -24363,7 +23532,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 98304, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_assistant_prefill": true, @@ -24377,7 +23545,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "supports_assistant_prefill": true, @@ -24391,7 +23558,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 229376, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_assistant_prefill": true, @@ -24404,7 +23570,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - standard version with 2560 dimensions" }, @@ -24416,7 +23581,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - large version with 2048 dimensions" }, @@ -24428,7 +23592,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - text-240915 version with 4096 dimensions" }, @@ -24440,7 +23603,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - text-250515 version with 2048 dimensions" }, @@ -24452,7 +23614,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - text-240715 version with 2560 dimensions" }, @@ -24565,7 +23726,6 @@ "mode": "search" }, "sail/moonshotai/Kimi-K3": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 2.5e-06, @@ -24587,7 +23747,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/zai-org/GLM-5.3": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9.8e-07, @@ -24609,7 +23768,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/zai-org/GLM-5.3-Flash": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.1e-07, @@ -24631,7 +23789,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/deepseek-ai/DeepSeek-V4-Pro-0813": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9.2e-07, @@ -24653,7 +23810,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9e-08, @@ -24675,7 +23831,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/deepseek-ai/DeepSeek-V4.1-Flash": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9e-08, @@ -24697,7 +23852,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/moonshotai/Kimi-K2.6": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-06, @@ -24720,7 +23874,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/google/gemma-4-31B-it": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 4e-07, @@ -24743,7 +23896,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/nvidia/Gemma-4-31B-IT-NVFP4": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.4e-07, @@ -24766,7 +23918,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/google/gemma-4-12B-it": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 3e-07, @@ -24788,7 +23939,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6e-08, @@ -24804,7 +23954,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/Qwen/Qwen3.6-35B-A3B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 5e-08, @@ -25043,7 +24192,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -25052,7 +24200,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "metadata": { "notes": "'supports_image_input' is a deprecated field. Use 'supports_embedding_image_input' instead." }, @@ -25065,7 +24212,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "supports_embedding_image_input": true @@ -25074,7 +24220,6 @@ "input_cost_per_token": 0.0001, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "supports_embedding_image_input": true @@ -25085,7 +24230,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.12e-07, "supports_function_calling": true, @@ -25101,7 +24245,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.84e-07, "supports_function_calling": true, @@ -25117,7 +24260,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 4.2e-06, "output_cost_per_token_flex": 2.1e-06, @@ -25134,7 +24276,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -25157,7 +24298,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -25185,7 +24325,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -25201,7 +24340,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -25220,7 +24358,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -25240,7 +24377,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -25257,7 +24393,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -25288,7 +24423,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -25319,7 +24453,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -25359,7 +24492,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -25393,7 +24525,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.3e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", @@ -25406,7 +24537,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", @@ -25418,7 +24548,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md", @@ -27684,14 +26813,12 @@ "litellm_provider": "featherless_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat" }, "featherless_ai/featherless-ai/Qwerky-QwQ-32B": { "litellm_provider": "featherless_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat" }, "fireworks-ai-4.1b-to-16b": { @@ -27739,7 +26866,6 @@ "input_cost_per_token": 1.6e-08, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -27749,7 +26875,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/pricing", @@ -27762,7 +26887,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 20480, - "max_tokens": 20480, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://fireworks.ai/pricing", @@ -27774,7 +26898,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 160000, - "max_tokens": 160000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://fireworks.ai/pricing", @@ -27786,7 +26909,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 20480, - "max_tokens": 20480, "mode": "chat", "output_cost_per_token": 2.19e-06, "source": "https://fireworks.ai/pricing", @@ -27798,7 +26920,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -27810,7 +26931,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/models/fireworks/deepseek-v3-0324", @@ -27822,7 +26942,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://fireworks.ai/pricing", @@ -27835,7 +26954,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://fireworks.ai/pricing", @@ -27848,7 +26966,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://fireworks.ai/models/fireworks/deepseek-v3p2", @@ -27863,7 +26980,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -27882,7 +26998,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.96e-06, "output_cost_per_token_priority": 4.95e-06, @@ -27902,7 +27017,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.96e-06, "output_cost_per_token_priority": 4.95e-06, @@ -27918,7 +27032,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -27931,7 +27044,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 96000, - "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 2.19e-06, "source": "https://fireworks.ai/models/fireworks/glm-4p5", @@ -27945,7 +27057,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 96000, - "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://artificialanalysis.ai/models/glm-4-5-air", @@ -27960,7 +27071,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "source": "https://fireworks.ai/pricing", "supports_function_calling": true, @@ -27974,7 +27084,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://fireworks.ai/models/fireworks/glm-4p7", @@ -27989,7 +27098,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28008,7 +27116,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -28028,7 +27135,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 7.2e-07, @@ -28045,7 +27151,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28060,7 +27165,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct", @@ -28073,7 +27177,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://app.fireworks.ai/models/fireworks/kimi-k2-instruct-0905", @@ -28086,7 +27189,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://fireworks.ai/pricing", @@ -28101,7 +27203,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28118,7 +27219,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28138,7 +27238,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28154,7 +27253,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28167,7 +27265,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://fireworks.ai/pricing", @@ -28180,7 +27277,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://fireworks.ai/pricing", @@ -28194,7 +27290,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://fireworks.ai/pricing", @@ -28207,7 +27302,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://fireworks.ai/pricing", @@ -28220,7 +27314,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -28233,7 +27326,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://fireworks.ai/pricing", @@ -28245,7 +27337,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://fireworks.ai/pricing", @@ -28258,7 +27349,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 196608, "max_output_tokens": 204800, - "max_tokens": 204800, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/models/fireworks/minimax-m2p1", @@ -28274,7 +27364,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.8e-06, @@ -28290,7 +27379,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/pricing", @@ -28303,7 +27391,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -28316,7 +27403,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -28329,7 +27415,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28343,7 +27428,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28359,7 +27443,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://fireworks.ai/models/fireworks/glm-4p7", @@ -28374,7 +27457,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28390,7 +27472,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28409,7 +27490,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -28429,7 +27509,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 7.2e-07, @@ -28446,7 +27525,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28462,7 +27540,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28479,7 +27556,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28497,7 +27573,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28516,7 +27591,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28534,7 +27608,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28550,7 +27623,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 196608, "max_output_tokens": 204800, - "max_tokens": 204800, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/models/fireworks/minimax-m2p1", @@ -28566,7 +27638,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.8e-06, @@ -28583,7 +27654,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -28597,7 +27667,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28606,7 +27675,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28615,7 +27683,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28624,7 +27691,6 @@ "input_cost_per_token": 1.6e-08, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28633,7 +27699,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 3e-08, @@ -28661,7 +27726,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "input_cost_per_token": 1.26e-06, "output_cost_per_token": 3.96e-06, "cache_read_input_token_cost": 2.34e-07, @@ -28689,7 +27753,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 4e-07, "supports_prompt_caching": false, @@ -28712,7 +27775,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, @@ -28739,7 +27801,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "input_cost_per_token": 5e-07, "output_cost_per_token": 1.5e-06, "cache_read_input_token_cost": 2.5e-07, @@ -28763,7 +27824,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 196608, "max_output_tokens": 196608, - "max_tokens": 196608, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-08, @@ -28787,7 +27847,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 202752, "max_output_tokens": 202752, - "max_tokens": 202752, "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, @@ -28814,7 +27873,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 9e-07, @@ -28827,7 +27885,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, @@ -28840,7 +27897,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_batches": 3e-06, @@ -28854,7 +27910,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_system_messages": true, @@ -28866,7 +27921,6 @@ "litellm_provider": "openai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_system_messages": true, @@ -28878,7 +27932,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_system_messages": true, @@ -28890,7 +27943,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "source": "OpenAI needs to add pricing for this ft model, will be updated when added by OpenAI. Defaulting to base model pricing", @@ -28907,7 +27959,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -28928,7 +27979,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -28948,7 +27998,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -28970,7 +28019,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, @@ -28991,7 +28039,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.2e-06, "output_cost_per_token_batches": 1.6e-06, @@ -29012,7 +28059,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, "output_cost_per_token_batches": 4e-07, @@ -29033,7 +28079,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 1.6e-05, "output_cost_per_token_batches": 8e-06, @@ -29056,7 +28101,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29113,7 +28157,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, @@ -29171,7 +28214,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -29225,7 +28267,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -29277,7 +28318,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -29323,7 +28363,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -29372,7 +28411,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -29412,7 +28450,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -29471,7 +28508,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -29531,7 +28567,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29587,7 +28622,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -29625,7 +28659,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -29680,7 +28713,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -29727,7 +28759,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29774,7 +28805,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -29820,7 +28850,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -29864,7 +28893,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -29916,7 +28944,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -29978,7 +29005,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_image": 0.00012, "output_cost_per_token": 1.2e-05, @@ -30031,7 +29057,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -30084,7 +29109,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -30145,7 +29169,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -30192,7 +29215,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -30255,7 +29277,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -30316,7 +29337,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -30380,7 +29400,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -30436,7 +29455,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 7.5e-06, "output_cost_per_token": 7.5e-06, @@ -30484,7 +29502,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -30527,7 +29544,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_image": 0.00012, "output_cost_per_token": 1.2e-05, @@ -30580,7 +29596,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -30637,7 +29652,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -30670,7 +29684,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_reasoning_token": 2.5e-06, @@ -30708,7 +29721,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, @@ -30755,7 +29767,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -30779,7 +29790,6 @@ "input_cost_per_token_batches": 1.2e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30796,7 +29806,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30814,7 +29823,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30833,7 +29841,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30852,7 +29859,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30866,7 +29872,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30878,7 +29883,6 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "gemini", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30897,7 +29901,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30918,7 +29921,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -30964,7 +29966,6 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "image_generation", "output_cost_per_image": 0.045, "output_cost_per_image_token": 6e-05, @@ -31016,7 +30017,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -31076,7 +30076,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -31093,7 +30092,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "rpm": 1000, @@ -31130,7 +30128,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "rpm": 1000, @@ -31176,7 +30173,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -31239,7 +30235,6 @@ "supports_reasoning": false, "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, @@ -31291,7 +30286,6 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -31340,7 +30334,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -31383,7 +30376,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.045, "output_cost_per_image_token": 6e-05, @@ -31425,7 +30417,6 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -31464,7 +30455,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, @@ -31509,7 +30499,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -31566,7 +30555,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -31625,7 +30613,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -31684,7 +30671,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, @@ -31714,7 +30700,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -31768,7 +30753,6 @@ "litellm_provider": "gemini", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -31811,7 +30795,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -31873,7 +30856,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -31936,7 +30918,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -31998,7 +30979,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32063,7 +31043,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32125,7 +31104,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32188,7 +31166,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32246,7 +31223,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32284,7 +31260,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -32348,7 +31323,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -32417,7 +31391,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -32462,7 +31435,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32497,7 +31469,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32560,7 +31531,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32621,7 +31591,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32685,7 +31654,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32741,7 +31709,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 7.5e-06, "output_cost_per_token": 7.5e-06, @@ -32788,7 +31755,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -32823,7 +31789,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "metadata": { "notes": "Rate limits not documented for gemini-exp-1114. Assuming same as gemini-1.5-pro.", "supports_tool_choice": true @@ -32846,7 +31811,6 @@ "litellm_provider": "gemini", "max_input_tokens": 2097152, "max_output_tokens": 8192, - "max_tokens": 8192, "metadata": { "notes": "Rate limits not documented for gemini-exp-1206. Assuming same as gemini-1.5-pro.", "supports_tool_choice": true @@ -32867,7 +31831,7 @@ "input_cost_per_token": 3.5e-07, "litellm_provider": "gemini", "max_output_tokens": 8192, - "max_tokens": 8192, + "max_input_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.05e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -32881,7 +31845,7 @@ "input_cost_per_token": 3.5e-07, "litellm_provider": "gemini", "max_output_tokens": 8192, - "max_tokens": 8192, + "max_input_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.05e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -32905,7 +31869,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_character": 0, "output_cost_per_character_above_128k_tokens": 0, @@ -32925,7 +31888,6 @@ "litellm_provider": "gemini", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -32941,7 +31903,6 @@ "litellm_provider": "gemini", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -32977,7 +31938,6 @@ "litellm_provider": "gemini", "max_input_tokens": 32767, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_character": 0, "output_cost_per_character_above_128k_tokens": 0, @@ -32996,7 +31956,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_image": 0.04, "output_cost_per_token": 0, @@ -33021,7 +31980,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -33045,7 +32003,6 @@ "deprecation_date": "2026-10-22", "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -33062,7 +32019,6 @@ "deprecation_date": "2026-10-22", "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -33078,7 +32034,6 @@ "deprecation_date": "2026-10-22", "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.05, "output_cost_per_second_1080p": 0.08, @@ -33093,7 +32048,6 @@ "gemini/veo-3.1-fast-generate-001": { "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -33109,7 +32063,6 @@ "gemini/veo-3.1-generate-001": { "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -33128,7 +32081,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33151,7 +32103,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33174,7 +32125,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 5e-06, "supported_endpoints": [ @@ -33195,7 +32145,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06, @@ -33220,7 +32169,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33243,7 +32191,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33266,7 +32213,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33288,7 +32234,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33311,7 +32256,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33333,7 +32277,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33356,7 +32299,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33375,7 +32317,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33392,7 +32333,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33400,7 +32340,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33408,7 +32347,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33416,7 +32354,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33424,7 +32361,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33434,7 +32370,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "supports_function_calling": true, @@ -33447,7 +32382,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "supports_function_calling": true, @@ -33463,7 +32397,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -33473,7 +32406,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -33483,7 +32415,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33492,7 +32423,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -33502,7 +32432,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33511,7 +32440,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33523,7 +32451,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://github.blog/changelog/2026-10-19-github-copilot-models-deprecated-on-october-19-2026", @@ -33538,7 +32465,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "supported_endpoints": [ @@ -33559,7 +32485,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -33581,7 +32506,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 4.5e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33603,7 +32527,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -33627,7 +32550,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_200k_tokens": 1.8e-06, @@ -33651,7 +32573,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -33676,7 +32597,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -33698,9 +32618,8 @@ "input_cost_per_token": 1e-05, "input_cost_per_token_above_272k_tokens": 2e-05, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -33722,9 +32641,8 @@ "input_cost_per_token": 1e-07, "input_cost_per_token_above_272k_tokens": 2e-07, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -33746,9 +32664,8 @@ "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -33770,9 +32687,8 @@ "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -33792,7 +32708,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 917504, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33810,7 +32725,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.2e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33825,19 +32739,16 @@ "github_copilot/text-embedding-3-small": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding" }, "github_copilot/text-embedding-3-small-inference": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding" }, "github_copilot/text-embedding-ada-002": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding" }, "chatgpt/gpt-6-sol": { @@ -33849,7 +32760,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33870,7 +32780,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33891,7 +32800,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33912,7 +32820,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33929,7 +32836,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.5", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -33949,7 +32855,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.6-luna", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -33969,7 +32874,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.6-sol", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -33989,7 +32893,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.6-terra", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34008,7 +32911,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34028,7 +32930,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34046,7 +32947,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34064,7 +32964,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34078,7 +32977,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34093,7 +32991,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34109,7 +33006,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34124,7 +33020,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34144,7 +33039,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34159,7 +33053,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34175,7 +33068,6 @@ "litellm_provider": "gigachat", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true, @@ -34186,7 +33078,6 @@ "litellm_provider": "gigachat", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true, @@ -34198,7 +33089,6 @@ "litellm_provider": "gigachat", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true, @@ -34209,7 +33099,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -34218,7 +33107,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -34227,7 +33115,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 4096, - "max_tokens": 4096, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2560 @@ -34236,7 +33123,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 4096, - "max_tokens": 4096, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048 @@ -34246,7 +33132,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_function_calling": true, @@ -34258,7 +33143,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -34269,7 +33153,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -34280,7 +33163,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -34291,7 +33173,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.4e-05, "supports_function_calling": true @@ -34301,7 +33182,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true @@ -34311,7 +33191,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true @@ -34321,7 +33200,6 @@ "litellm_provider": "gmi", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -34332,7 +33210,6 @@ "litellm_provider": "gmi", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -34343,7 +33220,6 @@ "litellm_provider": "gmi", "max_input_tokens": 163840, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-07, "supports_function_calling": true @@ -34353,7 +33229,6 @@ "litellm_provider": "gmi", "max_input_tokens": 163840, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 8.8e-07, "supports_function_calling": true @@ -34363,7 +33238,6 @@ "litellm_provider": "gmi", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -34375,7 +33249,6 @@ "litellm_provider": "gmi", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -34384,7 +33257,6 @@ "litellm_provider": "gmi", "max_input_tokens": 196608, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -34416,7 +33288,6 @@ "litellm_provider": "baseten", "max_input_tokens": 200000, "max_output_tokens": 200000, - "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -34461,7 +33332,6 @@ "litellm_provider": "baseten", "max_input_tokens": 128072, "max_output_tokens": 128072, - "max_tokens": 128072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inference.baseten.co/v1/models", @@ -34491,7 +33361,6 @@ "litellm_provider": "gmi", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-06, "supports_vision": true @@ -34501,7 +33370,6 @@ "litellm_provider": "gmi", "max_input_tokens": 202752, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2e-06 }, @@ -34514,7 +33382,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.9e-07, "output_cost_per_token_batches": 1.5e-07, @@ -34536,7 +33403,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 3.8e-07, "output_cost_per_token_batches": 1.9e-07, @@ -34558,7 +33424,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-08, "output_cost_per_token_batches": 4e-08, @@ -34590,7 +33455,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -34630,7 +33494,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -34664,7 +33527,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -34696,7 +33558,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -34717,7 +33578,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://developers.openai.com/api/docs/pricing", @@ -34733,7 +33593,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, @@ -34751,7 +33610,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 2e-06, @@ -34768,7 +33626,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "supports_prompt_caching": true, @@ -34781,7 +33638,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06, "source": "https://developers.openai.com/api/docs/pricing" @@ -34791,7 +33647,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, "max_output_tokens": 4097, - "max_tokens": 4097, "mode": "completion", "output_cost_per_token": 2e-06 }, @@ -34801,7 +33656,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "supports_function_calling": true, @@ -34816,7 +33670,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -34832,7 +33685,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -34847,7 +33699,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -34865,7 +33716,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_batches": 1.5e-05, @@ -34887,7 +33737,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -34931,7 +33780,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_priority": 1.4e-05, @@ -34974,7 +33822,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, @@ -35018,7 +33865,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_priority": 2.8e-06, @@ -35062,7 +33908,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -35100,7 +33945,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_priority": 8e-07, @@ -35137,7 +33981,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -35160,7 +34003,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -35184,7 +34026,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 1.7e-05, @@ -35209,7 +34050,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 1.7e-05, @@ -35231,7 +34071,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -35249,7 +34088,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -35267,7 +34105,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -35304,7 +34141,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -35339,7 +34175,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -35377,7 +34212,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35415,7 +34249,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35452,7 +34285,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35493,7 +34325,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -35523,7 +34354,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 1e-06, @@ -35550,7 +34380,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 6e-07, @@ -35570,7 +34399,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35588,7 +34416,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -35615,7 +34442,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -35696,7 +34522,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -35723,7 +34548,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -36277,7 +35101,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -36323,7 +35146,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5-codex", @@ -36358,7 +35180,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -36408,7 +35229,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex-mini", @@ -36439,7 +35259,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex-max", @@ -36471,7 +35290,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex", @@ -36503,7 +35321,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-chat-latest", @@ -36538,7 +35355,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -36590,7 +35406,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -36641,7 +35456,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.2-codex", @@ -36673,7 +35487,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.2-chat-latest", @@ -36708,7 +35521,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -36758,7 +35570,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -36800,7 +35611,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -36840,10 +35650,14 @@ "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_12hr": 2.5e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 2.5e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": 0.0003, + "cache_creation_input_token_cost_above_12hr_batches": 1.25e-05, "cache_creation_input_token_cost_above_12hr_flex": 1.25e-05, "cache_creation_input_token_cost_above_12hr_priority": 5e-05, + "cache_creation_input_token_cost_above_12hr_ultrafast": 0.00015, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, @@ -36876,7 +35690,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -36931,8 +35744,10 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-06, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -36961,7 +35776,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -37014,8 +35828,10 @@ "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_12hr": 2.5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 2.5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06, + "cache_creation_input_token_cost_above_12hr_batches": 1.25e-07, "cache_creation_input_token_cost_above_12hr_flex": 1.25e-07, "cache_creation_input_token_cost_above_12hr_priority": 5e-07, "cache_creation_input_token_cost_decisions": 1.25e-07, @@ -37050,7 +35866,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_decisions": 0.0, @@ -37136,7 +35951,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37186,8 +36000,10 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_12hr": 1e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 1e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05, + "cache_creation_input_token_cost_above_12hr_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_priority": 2e-05, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, @@ -37216,7 +36032,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37268,8 +36083,10 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-06, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -37298,7 +36115,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -37349,8 +36165,10 @@ "cache_creation_input_token_cost": 2.5e-07, "cache_creation_input_token_cost_above_12hr": 5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-07, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-07, "cache_creation_input_token_cost_above_12hr_priority": 1e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, @@ -37379,7 +36197,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, @@ -37437,7 +36254,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, @@ -37476,7 +36292,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, @@ -37515,7 +36330,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, @@ -37553,7 +36367,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37592,7 +36405,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37626,7 +36438,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "reasoning_effort_levels": [ @@ -37672,7 +36483,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -37735,7 +36545,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -37791,7 +36600,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -37843,7 +36651,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -37902,7 +36709,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -37960,7 +36766,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -38011,7 +36816,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -38064,7 +36868,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -38120,7 +36923,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_flex": 2.25e-06, @@ -38175,7 +36977,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_flex": 2.25e-06, @@ -38228,7 +37029,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, @@ -38280,7 +37080,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, @@ -38327,7 +37126,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 272000, - "max_tokens": 272000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -38370,7 +37168,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 272000, - "max_tokens": 272000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -38419,7 +37216,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -38465,7 +37261,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -38501,7 +37296,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5-chat-latest", @@ -38534,7 +37328,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -38578,7 +37371,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.3-chat-latest", @@ -38615,7 +37407,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -38667,7 +37458,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -38717,7 +37507,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -38767,7 +37556,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -38853,7 +37641,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -38888,7 +37675,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -38923,7 +37709,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -38958,7 +37743,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -38995,7 +37779,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -39033,7 +37816,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -39069,7 +37851,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -39095,7 +37876,6 @@ }, "gradient_ai/alibaba-qwen3-32b": { "litellm_provider": "gradient_ai", - "max_tokens": 40960, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -39110,7 +37890,6 @@ "gradient_ai/anthropic-claude-3-opus": { "input_cost_per_token": 1.5e-05, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 7.5e-05, "supported_endpoints": [ @@ -39126,7 +37905,6 @@ "gradient_ai/anthropic-claude-3.5-haiku": { "input_cost_per_token": 8e-07, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 4e-06, "supported_endpoints": [ @@ -39142,7 +37920,6 @@ "gradient_ai/anthropic-claude-3.5-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.5e-05, "supported_endpoints": [ @@ -39158,7 +37935,6 @@ "gradient_ai/anthropic-claude-3.7-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.5e-05, "supported_endpoints": [ @@ -39174,7 +37950,6 @@ "gradient_ai/deepseek-r1-distill-llama-70b": { "input_cost_per_token": 9.9e-07, "litellm_provider": "gradient_ai", - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 9.9e-07, "supported_endpoints": [ @@ -39190,7 +37965,6 @@ "gradient_ai/llama3-8b-instruct": { "input_cost_per_token": 2e-07, "litellm_provider": "gradient_ai", - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 2e-07, "supported_endpoints": [ @@ -39206,7 +37980,6 @@ "gradient_ai/llama3.3-70b-instruct": { "input_cost_per_token": 6.5e-07, "litellm_provider": "gradient_ai", - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6.5e-07, "supported_endpoints": [ @@ -39222,7 +37995,6 @@ "gradient_ai/mistral-nemo-instruct-2407": { "input_cost_per_token": 3e-07, "litellm_provider": "gradient_ai", - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 3e-07, "supported_endpoints": [ @@ -39237,7 +38009,6 @@ }, "gradient_ai/openai-gpt-4o": { "litellm_provider": "gradient_ai", - "max_tokens": 16384, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -39251,7 +38022,6 @@ }, "gradient_ai/openai-gpt-4o-mini": { "litellm_provider": "gradient_ai", - "max_tokens": 16384, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -39266,7 +38036,6 @@ "gradient_ai/openai-o3": { "input_cost_per_token": 2e-06, "litellm_provider": "gradient_ai", - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "supported_endpoints": [ @@ -39282,7 +38051,6 @@ "gradient_ai/openai-o3-mini": { "input_cost_per_token": 1.1e-06, "litellm_provider": "gradient_ai", - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supported_endpoints": [ @@ -39298,7 +38066,6 @@ "lemonade/Qwen3-Coder-30B-A3B-Instruct-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", @@ -39310,7 +38077,6 @@ "lemonade/gpt-oss-20b-mxfp4-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "chat", @@ -39322,7 +38088,6 @@ "lemonade/gpt-oss-120b-mxfp-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "chat", @@ -39334,7 +38099,6 @@ "lemonade/Gemma-3-4b-it-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", @@ -39346,7 +38110,6 @@ "lemonade/Qwen3-4B-Instruct-2507-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", @@ -39360,7 +38123,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.4e-07, "supports_function_calling": true, @@ -39372,7 +38134,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.4e-07, "supports_function_calling": true, @@ -39386,7 +38147,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 1000000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.25e-05, "supports_function_calling": true, @@ -39400,7 +38160,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, "supports_function_calling": true, @@ -39414,7 +38173,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://console.groq.com/docs/model/llama-guard-3-8b" @@ -39424,7 +38183,6 @@ "litellm_provider": "groq", "max_input_tokens": 512, "max_output_tokens": 512, - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 3e-08, "source": "https://console.groq.com/docs/models" @@ -39434,7 +38192,6 @@ "litellm_provider": "groq", "max_input_tokens": 512, "max_output_tokens": 512, - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 4e-08, "source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m" @@ -39445,7 +38202,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "search_context_cost_per_query": { @@ -39466,7 +38222,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "search_context_cost_per_query": { @@ -39487,7 +38242,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "search_context_cost_per_query": { @@ -39506,7 +38260,6 @@ "litellm_provider": "groq", "max_input_tokens": 4000, "max_output_tokens": 50000, - "max_tokens": 50000, "mode": "audio_speech", "source": "https://console.groq.com/docs/model/canopylabs/orpheus-v1-english" }, @@ -39515,7 +38268,6 @@ "litellm_provider": "groq", "max_input_tokens": 4000, "max_output_tokens": 50000, - "max_tokens": 50000, "mode": "audio_speech", "source": "https://console.groq.com/docs/models" }, @@ -39533,7 +38285,6 @@ }, "heroku/claude-3-5-haiku": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39543,7 +38294,6 @@ }, "heroku/claude-3-5-sonnet-latest": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39553,7 +38303,6 @@ }, "heroku/claude-3-7-sonnet": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39563,7 +38312,6 @@ }, "heroku/claude-4-sonnet": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39609,7 +38357,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39622,7 +38369,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -39635,7 +38381,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39648,7 +38393,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39661,7 +38405,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -39674,7 +38417,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "supports_function_calling": true, @@ -39687,7 +38429,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_function_calling": true, @@ -39700,7 +38441,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -39713,7 +38453,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "supports_function_calling": true, @@ -39726,7 +38465,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39739,7 +38477,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39752,7 +38489,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39765,7 +38501,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39778,7 +38513,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39791,7 +38525,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39804,7 +38537,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -39817,7 +38549,6 @@ "litellm_provider": "ai21", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 3e-06 }, @@ -39826,7 +38557,6 @@ "litellm_provider": "ai21", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 1e-05 }, @@ -39835,7 +38565,6 @@ "litellm_provider": "ai21", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 1.5e-05 }, @@ -39844,7 +38573,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39854,7 +38582,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39864,7 +38591,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39874,7 +38600,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39884,7 +38609,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39894,7 +38618,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39904,7 +38627,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39914,7 +38636,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39924,7 +38645,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39934,7 +38654,6 @@ "litellm_provider": "jina_ai", "max_input_tokens": 1024, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://api.jina.ai/v1/models" @@ -39954,7 +38673,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -39991,7 +38709,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -40019,7 +38736,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 7e-06, "supports_function_calling": false, @@ -40032,7 +38748,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -40045,7 +38760,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -40059,7 +38773,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -40072,7 +38785,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-06, "supports_function_calling": false, @@ -40085,7 +38797,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -40098,7 +38809,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -40112,7 +38822,6 @@ "litellm_provider": "inception", "max_input_tokens": 128000, "max_output_tokens": 50000, - "max_tokens": 50000, "mode": "chat", "output_cost_per_token": 7.5e-07, "supports_function_calling": true, @@ -40127,7 +38836,6 @@ "litellm_provider": "inception", "max_input_tokens": 260000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-07, "source": "https://docs.inceptionlabs.ai/get-started/models", @@ -40142,7 +38850,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40163,7 +38871,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40184,7 +38892,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40205,7 +38913,7 @@ "input_cost_per_token": 1.72e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40226,7 +38934,7 @@ "input_cost_per_token": 1.72e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40247,7 +38955,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://inferenceapis.com/pricing", @@ -40268,7 +38976,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40289,7 +38997,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://inferenceapis.com/pricing", @@ -40310,7 +39018,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40331,7 +39039,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40352,7 +39060,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40373,7 +39081,7 @@ "input_cost_per_token": 1.72e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40394,7 +39102,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://inferenceapis.com/pricing", @@ -40415,7 +39123,7 @@ "input_cost_per_token": 1.82e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 5.72e-06, "source": "https://inferenceapis.com/pricing", @@ -40436,7 +39144,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://inferenceapis.com/pricing", @@ -40456,7 +39164,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://inferenceapis.com/pricing", @@ -40475,7 +39183,7 @@ "input_cost_per_token": 4e-08, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://inferenceapis.com/pricing", @@ -40495,7 +39203,7 @@ "input_cost_per_token": 8.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.42e-06, "source": "https://inferenceapis.com/pricing", @@ -40515,7 +39223,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40534,7 +39242,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40553,7 +39261,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40572,7 +39280,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40591,7 +39299,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40610,7 +39318,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40630,7 +39338,7 @@ "input_cost_per_token": 3.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://inferenceapis.com/pricing", @@ -40651,7 +39359,7 @@ "input_cost_per_token": 3.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://inferenceapis.com/pricing", @@ -40672,7 +39380,7 @@ "input_cost_per_token": 8.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.42e-06, "source": "https://inferenceapis.com/pricing", @@ -40692,7 +39400,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://inferenceapis.com/pricing", @@ -40711,7 +39419,7 @@ "input_cost_per_token": 4e-08, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://inferenceapis.com/pricing", @@ -40730,7 +39438,7 @@ "input_cost_per_token": 2.6e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40750,7 +39458,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40770,7 +39478,7 @@ "input_cost_per_token": 2.6e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40789,7 +39497,7 @@ "input_cost_per_token": 2.6e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40809,7 +39517,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40830,7 +39538,7 @@ "input_cost_per_token": 1.82e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 5.72e-06, "source": "https://inferenceapis.com/pricing", @@ -40851,7 +39559,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://inferenceapis.com/pricing", @@ -40873,7 +39581,6 @@ "litellm_provider": "text-completion-inception", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 7.5e-07 }, @@ -40882,7 +39589,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -40896,7 +39602,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -40910,7 +39615,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -40924,7 +39628,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -40937,7 +39640,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -40950,7 +39652,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -40963,7 +39664,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -40976,7 +39676,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -40989,7 +39688,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -41002,7 +39700,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41015,7 +39712,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 16384, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41028,7 +39724,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -41041,7 +39736,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -41054,7 +39748,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -41067,7 +39760,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -41080,7 +39772,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-08, "supports_function_calling": true, @@ -41094,7 +39785,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-08, "supports_function_calling": true, @@ -41107,7 +39797,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -41120,7 +39809,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41133,7 +39821,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41178,14 +39865,12 @@ "max-x-max/50-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.036 }, "max-x-max/max-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.072 }, @@ -41281,7 +39966,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_character": 1.5e-05, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -41292,7 +39976,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_character": 1e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -41303,7 +39986,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -41313,7 +39995,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.56e-06, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -41324,7 +40005,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -41338,7 +40018,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -41353,7 +40032,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.2e-07, "output_cost_per_token_batches": 1.1e-07, @@ -41367,7 +40045,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.6e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -41381,7 +40058,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", @@ -41394,7 +40070,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", @@ -41407,7 +40082,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -41422,7 +40096,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -41436,7 +40109,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -41447,7 +40119,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -41459,7 +40130,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9.7e-07, "output_cost_per_token_batches": 4.85e-07, @@ -41482,7 +40152,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 10000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_batches": 3.3e-07, @@ -41504,7 +40173,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.25e-06, "search_context_cost_per_query": { @@ -41545,7 +40213,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.25e-06, "search_context_cost_per_query": { @@ -41586,7 +40253,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "search_context_cost_per_query": { @@ -41627,7 +40293,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.25e-06, "search_context_cost_per_query": { @@ -41668,7 +40333,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "search_context_cost_per_query": { @@ -41725,7 +40389,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-05, "search_context_cost_per_query": { @@ -41766,7 +40429,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 128000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41782,7 +40444,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 128000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41798,7 +40459,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 1000000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41815,7 +40475,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 10000000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41837,7 +40496,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -41860,7 +40518,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -41884,7 +40541,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 6e-07, "output_cost_per_token_priority": 2.1e-06, @@ -42082,7 +40738,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -42104,7 +40759,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 40000, - "max_tokens": 40000, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, @@ -42126,7 +40780,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_batches": 1e-07, @@ -42148,7 +40801,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-07, "output_cost_per_token_batches": 5e-08, @@ -42170,7 +40822,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-07, "output_cost_per_token_batches": 7e-08, @@ -42190,7 +40841,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -42201,7 +40851,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -42212,7 +40861,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -42227,7 +40875,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, @@ -42247,7 +40894,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -42260,7 +40906,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -42275,7 +40920,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-08, "output_cost_per_token_batches": 2e-08, @@ -42295,7 +40939,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, "output_cost_per_token_batches": 1.5e-07, @@ -42312,7 +40955,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -42327,7 +40969,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -42342,7 +40983,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://mistral.ai/technology/", @@ -42355,7 +40995,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://docs.mistral.ai/models/devstral-small-2-25-12", @@ -42370,7 +41009,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://mistral.ai/news/devstral-2-vibe-cli", @@ -42385,7 +41023,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://mistral.ai/news/devstral-2-vibe-cli", @@ -42399,7 +41036,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12", @@ -42415,7 +41051,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12", @@ -42431,7 +41066,6 @@ "litellm_provider": "mistral", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12", @@ -42447,7 +41081,6 @@ "litellm_provider": "mistral", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12", @@ -42462,7 +41095,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "source": "https://docs.mistral.ai/models/mistral-embed-23-12" }, @@ -42472,7 +41104,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -42653,7 +41284,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://docs.mistral.ai/models/voxtral-small-25-07", @@ -42668,7 +41298,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://docs.mistral.ai/models/voxtral-small-25-07", @@ -42684,7 +41313,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42710,7 +41338,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42732,7 +41359,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "reasoning_effort_levels": [ @@ -42755,7 +41381,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42777,7 +41402,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42799,7 +41423,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42878,7 +41501,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", @@ -42895,7 +41517,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", @@ -42910,7 +41531,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding" }, "mistral/codestral-embed": { @@ -42918,7 +41538,6 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding" }, "mistral/codestral-embed-2505": { @@ -42926,7 +41545,6 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding" }, "mistral/mistral-large-4": { @@ -42934,7 +41552,7 @@ "input_cost_per_token": 6.8e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.09e-06, "reasoning_effort_levels": [ @@ -42955,7 +41573,7 @@ "input_cost_per_token": 6.8e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.09e-06, "reasoning_effort_levels": [ @@ -42977,7 +41595,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", @@ -42993,7 +41610,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", @@ -43009,7 +41625,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", @@ -43025,7 +41640,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43046,7 +41660,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43067,7 +41680,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43088,7 +41700,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43109,7 +41720,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-07, "supports_assistant_prefill": true, @@ -43123,7 +41733,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "reasoning_effort_levels": [ @@ -43144,7 +41753,6 @@ "litellm_provider": "mistral", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://mistral.ai/pricing", @@ -43160,7 +41768,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://mistral.ai/pricing", @@ -43176,7 +41783,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://mistral.ai/pricing", @@ -43192,7 +41798,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://mistral.ai/pricing", @@ -43208,7 +41813,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://mistral.ai/pricing", @@ -43224,7 +41828,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_assistant_prefill": true, @@ -43237,7 +41840,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://mistral.ai/technology/", @@ -43251,7 +41853,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-06, "supports_assistant_prefill": true, @@ -43268,7 +41869,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -43290,7 +41890,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -43310,7 +41909,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://platform.kimi.ai/docs/pricing/chat-k27-code", @@ -43328,7 +41926,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", @@ -43345,7 +41942,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://platform.kimi.ai/docs/pricing/chat-k26", @@ -43362,7 +41958,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ @@ -43383,7 +41978,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43396,7 +41990,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43410,7 +42003,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43423,7 +42015,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43437,7 +42028,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43450,7 +42040,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43464,7 +42053,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43477,7 +42065,6 @@ "litellm_provider": "morph", "max_input_tokens": 16000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": false, @@ -43491,7 +42078,6 @@ "litellm_provider": "morph", "max_input_tokens": 16000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.9e-06, "supports_function_calling": false, @@ -43509,7 +42095,6 @@ "input_cost_per_video_per_second_above_8s_interval": 0.001, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -43533,7 +42118,6 @@ "input_cost_per_video_per_second_above_8s_interval": 0.001, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -43693,7 +42277,6 @@ ] }, "nebius/deepseek-ai/DeepSeek-R1": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 8e-07, @@ -43705,7 +42288,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-0528": { - "max_tokens": 164000, "max_input_tokens": 164000, "max_output_tokens": 164000, "input_cost_per_token": 8e-07, @@ -43717,7 +42299,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2.5e-07, @@ -43728,7 +42309,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 5e-07, @@ -43739,7 +42319,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3-0324": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 5e-07, @@ -43750,7 +42329,6 @@ "source": "https://nebius.com/prices" }, "nebius/google/gemma-3-27b-it": { - "max_tokens": 110000, "max_input_tokens": 110000, "max_output_tokens": 110000, "input_cost_per_token": 1e-07, @@ -43762,7 +42340,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, @@ -43773,7 +42350,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Llama-Guard-3-8B": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-08, @@ -43783,7 +42359,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-8B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-08, @@ -43794,7 +42369,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.3e-07, @@ -43805,7 +42379,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-405B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1e-06, @@ -43816,7 +42389,6 @@ "source": "https://nebius.com/prices" }, "nebius/mistralai/Mistral-Nemo-Instruct-2407": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 4e-08, @@ -43827,7 +42399,6 @@ "source": "https://nebius.com/prices" }, "nebius/NousResearch/Hermes-3-Llama-3.1-405B": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1e-06, @@ -43838,7 +42409,6 @@ "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 6e-07, @@ -43849,7 +42419,6 @@ "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.3-Nemotron-Super-49B-v1": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -43860,7 +42429,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-235B-A22B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2e-07, @@ -43871,7 +42439,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-32B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-07, @@ -43882,7 +42449,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-32B" }, "nebius/Qwen/Qwen3-30B-A3B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -43893,7 +42459,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-14B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 8e-08, @@ -43904,7 +42469,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-4B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 8e-08, @@ -43915,7 +42479,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/QwQ-32B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.5e-07, @@ -43927,7 +42490,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-72B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.3e-07, @@ -43938,7 +42500,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-32B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 6e-08, @@ -43949,7 +42510,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-Coder-7B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-08, @@ -43960,7 +42520,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-VL-72B-Instruct": { - "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, "input_cost_per_token": 2.5e-07, @@ -43972,7 +42531,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/Qwen%2FQwen2.5-VL-72B-Instruct" }, "nebius/Qwen/Qwen2-VL-72B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, @@ -43984,7 +42542,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2-VL-7B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, @@ -43995,7 +42552,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V4-Flash": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.4e-07, @@ -44007,7 +42563,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash" }, "nebius/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_tokens": 1024000, "max_input_tokens": 1024000, "max_output_tokens": 1024000, "input_cost_per_token": 1.4e-07, @@ -44019,7 +42574,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731" }, "nebius/deepseek-ai/DeepSeek-V4-Pro": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.75e-06, @@ -44045,7 +42599,6 @@ "litellm_provider": "nebius", "max_input_tokens": 1048000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://tokenfactory.nebius.com/endpoints?modals=endpoint-details&model-id=deepseek-ai/DeepSeek-V4.1-Flash", @@ -44054,7 +42607,6 @@ "supports_vision": true }, "nebius/MiniMaxAI/MiniMax-M2.5": { - "max_tokens": 196608, "max_input_tokens": 196608, "max_output_tokens": 196608, "input_cost_per_token": 3e-07, @@ -44066,7 +42618,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M2.5" }, "nebius/MiniMaxAI/MiniMax-M3": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 3e-07, @@ -44079,7 +42630,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3" }, "nebius/moonshotai/Kimi-K2.6": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9.5e-07, @@ -44092,7 +42642,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6" }, "nebius/moonshotai/Kimi-K2.7-Code": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9.5e-07, @@ -44105,7 +42654,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" }, "nebius/moonshotai/Kimi-K3": { - "max_tokens": 1024000, "max_input_tokens": 1024000, "max_output_tokens": 1024000, "input_cost_per_token": 3e-06, @@ -44118,7 +42666,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3" }, "nebius/NousResearch/Hermes-4-405B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-06, @@ -44130,7 +42677,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B" }, "nebius/NousResearch/Hermes-4-70B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, @@ -44142,7 +42688,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-70B" }, "nebius/nvidia/Cosmos3-Super-Reasoner": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-07, @@ -44155,7 +42700,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/nvidia%2FCosmos3-Super-Reasoner" }, "nebius/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6e-07, @@ -44166,7 +42710,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FLlama-3_1-Nemotron-Ultra-253B-v1" }, "nebius/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 6e-08, @@ -44178,7 +42721,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B" }, "nebius/nvidia/Nemotron-3-Nano-Omni": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 6e-08, @@ -44191,7 +42733,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Nano-Omni" }, "nebius/nvidia/nemotron-3-super-120b-a12b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 3e-07, @@ -44203,7 +42744,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b" }, "nebius/nvidia/Nemotron-3-Ultra-550b-a55b": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1e-06, @@ -44215,7 +42755,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b" }, "nebius/nvidia/Nemotron-3_5-Lightning": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 6e-08, @@ -44227,7 +42766,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning" }, "nebius/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.5e-07, @@ -44239,7 +42777,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b" }, "nebius/openbmb/MiniCPM-V-4_5": { - "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, "input_cost_per_token": 6.58e-07, @@ -44250,7 +42787,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5" }, "nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2e-07, @@ -44261,7 +42797,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507" }, "nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-07, @@ -44272,7 +42807,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507" }, "nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.5e-07, @@ -44284,7 +42818,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-Next-80B-A3B-Thinking" }, "nebius/Qwen/Qwen3.5-397B-A17B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 6e-07, @@ -44306,7 +42839,6 @@ "supports_reasoning": true }, "nebius/zai-org/GLM-5.1": { - "max_tokens": 202752, "max_input_tokens": 202752, "max_output_tokens": 202752, "input_cost_per_token": 1.4e-06, @@ -44318,7 +42850,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1" }, "nebius/zai-org/GLM-5.2": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.4e-06, @@ -44333,7 +42864,7 @@ "input_cost_per_token": 1.4e-06, "litellm_provider": "nebius", "max_input_tokens": 1024000, - "max_tokens": 1024000, + "max_output_tokens": 1024000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3", @@ -44341,7 +42872,6 @@ "supports_reasoning": true }, "nebius/zai-org/GLM-5.3-Flash": { - "max_tokens": 1024000, "max_input_tokens": 1024000, "max_output_tokens": 1024000, "input_cost_per_token": 1.5e-07, @@ -44354,7 +42884,6 @@ "supports_vision": true }, "nebius/BAAI/bge-en-icl": { - "max_tokens": 32768, "max_input_tokens": 32768, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44363,7 +42892,6 @@ "source": "https://nebius.com/prices" }, "nebius/BAAI/bge-multilingual-gemma2": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44372,7 +42900,6 @@ "source": "https://nebius.com/prices" }, "nebius/intfloat/e5-mistral-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44381,7 +42908,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-Embedding-8B": { - "max_tokens": 40960, "max_input_tokens": 40960, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44398,7 +42924,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -44420,7 +42945,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.3e-07, "output_cost_per_token_batches": 1.2e-07, @@ -44441,7 +42965,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07, @@ -44464,7 +42987,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.5e-07, "output_cost_per_token_batches": 3.25e-07, @@ -44487,7 +43009,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -44510,7 +43031,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -44532,7 +43052,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 0.0006, "output_cost_per_token_batches": 0.0003, @@ -44566,7 +43085,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 0.0006, "output_cost_per_token_batches": 0.0003, @@ -44604,7 +43122,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -44645,7 +43162,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, "source": "https://developers.openai.com/api/docs/models/o3-deep-research", @@ -44678,7 +43194,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, "source": "https://developers.openai.com/api/docs/models/o3-deep-research", @@ -44715,7 +43230,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -44758,7 +43272,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -44779,7 +43292,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -44798,7 +43310,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -44836,7 +43347,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -44879,7 +43389,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -44907,7 +43416,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-06, "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research", @@ -44940,7 +43448,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-06, "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research", @@ -44977,7 +43484,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -45004,7 +43510,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45017,7 +43522,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45030,7 +43534,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.068e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45043,7 +43546,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45057,7 +43559,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45070,7 +43571,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45084,7 +43584,6 @@ "litellm_provider": "oci", "max_input_tokens": 10485760, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45097,7 +43596,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45110,7 +43608,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45123,7 +43620,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45136,7 +43632,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45149,7 +43644,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45162,7 +43656,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45175,7 +43668,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45188,7 +43680,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45201,7 +43692,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45221,7 +43711,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45240,7 +43729,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45260,7 +43748,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45274,7 +43761,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45296,7 +43782,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45308,7 +43793,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45321,7 +43805,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 9e-08, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45333,7 +43816,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45345,7 +43827,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45357,7 +43838,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45370,7 +43850,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45382,7 +43861,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45394,7 +43872,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45406,7 +43883,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [], @@ -45419,7 +43895,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45434,7 +43909,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45450,7 +43924,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45467,7 +43940,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45483,7 +43955,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45497,7 +43968,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45509,7 +43979,6 @@ "litellm_provider": "oci", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45524,7 +43993,6 @@ "litellm_provider": "oci", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45539,7 +44007,6 @@ "litellm_provider": "oci", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45554,7 +44021,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45567,7 +44033,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45579,7 +44044,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -45589,7 +44053,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45599,7 +44062,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -45609,7 +44071,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45619,7 +44080,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -45630,7 +44090,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45641,7 +44100,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45652,7 +44110,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "oci", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -45664,7 +44121,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": false @@ -45674,7 +44130,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45683,7 +44138,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45692,7 +44146,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45702,7 +44155,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45712,7 +44164,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45722,7 +44173,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45732,7 +44182,6 @@ "litellm_provider": "ollama", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45742,7 +44191,6 @@ "litellm_provider": "ollama", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45752,7 +44200,6 @@ "litellm_provider": "ollama", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45762,7 +44209,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45772,7 +44218,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45781,7 +44226,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45790,7 +44234,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45799,7 +44242,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45808,7 +44250,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45817,7 +44258,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45826,7 +44266,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45836,7 +44275,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45845,7 +44283,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45854,7 +44291,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45864,7 +44300,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45874,7 +44309,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45884,7 +44318,6 @@ "litellm_provider": "ollama", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45894,7 +44327,6 @@ "litellm_provider": "ollama", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45904,7 +44336,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45914,7 +44345,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45923,7 +44353,6 @@ "litellm_provider": "ollama", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45933,7 +44362,6 @@ "litellm_provider": "ollama", "max_input_tokens": 2048, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45942,7 +44370,6 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -45951,7 +44378,6 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -45969,7 +44395,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -45995,7 +44420,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07, "output_cost_per_token_batches": 1.5e-07, @@ -46015,7 +44439,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46037,7 +44460,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_batches": 1e-07, @@ -46056,7 +44478,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46070,7 +44491,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46083,7 +44503,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46096,7 +44515,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46109,7 +44527,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 127000, "max_output_tokens": 127000, - "max_tokens": 127000, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46122,7 +44539,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 118000, "max_output_tokens": 118000, - "max_tokens": 118000, "mode": "chat", "output_cost_per_token": 1.3e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46135,7 +44551,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46149,7 +44564,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.3e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46162,7 +44576,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 8.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46175,7 +44588,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 9.1e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46189,7 +44601,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.3e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46203,7 +44614,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46217,7 +44627,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46231,7 +44640,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.9e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46245,7 +44653,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46278,7 +44685,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-06 }, @@ -46287,7 +44693,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46296,7 +44701,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46305,7 +44709,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -46314,7 +44717,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07 }, @@ -46323,7 +44725,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46332,7 +44733,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46341,7 +44741,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46351,7 +44750,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46360,7 +44758,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46370,7 +44767,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46426,7 +44822,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-06, "search_context_cost_per_query": { @@ -46441,7 +44837,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 8e-06, @@ -46458,7 +44854,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.8e-06 }, @@ -46468,7 +44863,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 12000, "max_output_tokens": 12000, - "max_tokens": 12000, "mode": "chat", "output_cost_per_token": 1.8e-06 }, @@ -46477,7 +44871,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 200000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -46491,7 +44884,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-06, "search_context_cost_per_query": { @@ -46506,7 +44899,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 8e-06, "search_context_cost_per_query": { @@ -46522,7 +44915,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46532,7 +44924,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 12000, "max_output_tokens": 12000, - "max_tokens": 12000, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46541,7 +44932,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46553,7 +44943,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46565,7 +44954,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46577,7 +44965,6 @@ "litellm_provider": "publicai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46589,7 +44976,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46601,7 +44987,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46841,7 +45226,6 @@ "input_cost_per_token": 4e-09, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -46851,7 +45235,6 @@ "input_cost_per_token": 3e-08, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2560, @@ -46862,7 +45245,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46874,7 +45256,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46887,7 +45268,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46901,7 +45281,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.8e-06, "supports_function_calling": true, @@ -46916,7 +45295,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 8.8e-07, "output_cost_per_token_flex": 4.4e-07, @@ -46936,7 +45314,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46955,7 +45332,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46972,7 +45348,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -46988,7 +45363,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47004,7 +45378,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47020,7 +45393,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.236e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47036,7 +45408,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47052,7 +45423,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47068,7 +45438,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47086,7 +45455,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.66e-06, "output_cost_per_token_batches": 1.33e-06, @@ -47107,7 +45475,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -47162,7 +45529,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, "supports_tool_choice": true @@ -47172,7 +45538,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, "supports_tool_choice": true @@ -47182,7 +45547,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47192,7 +45556,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47202,7 +45565,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47212,7 +45574,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47222,7 +45583,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47232,7 +45592,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47242,7 +45601,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8086, "max_output_tokens": 8086, - "max_tokens": 8086, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47252,7 +45610,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8086, "max_output_tokens": 8086, - "max_tokens": 8086, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47262,7 +45619,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47272,7 +45628,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47282,7 +45637,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, "supports_tool_choice": true @@ -47411,7 +45765,6 @@ "mode": "chat", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_system_messages": true }, @@ -47566,7 +45919,6 @@ "mode": "chat", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true @@ -47587,7 +45939,6 @@ "mode": "chat", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_reasoning": true, "supports_system_messages": true }, @@ -47597,7 +45948,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -47607,7 +45957,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -47617,7 +45966,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -47627,7 +45975,6 @@ "litellm_provider": "cohere", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://cohere.com/pricing" @@ -47638,7 +45985,6 @@ "litellm_provider": "cohere", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://cohere.com/pricing" @@ -47669,7 +46015,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -47678,7 +46023,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -47687,7 +46031,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -47696,7 +46039,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -47705,7 +46047,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -47714,7 +46055,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -47723,7 +46063,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 196608, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://cloud.sambanova.ai/plans/pricing", @@ -47736,7 +46075,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7e-06, "source": "https://cloud.sambanova.ai/plans/pricing" @@ -47746,7 +46084,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount" }, @@ -47763,7 +46100,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.sambanova.ai/plans/pricing", @@ -47772,7 +46108,6 @@ "supports_tool_choice": true }, "sambanova/DeepSeek-V3.1": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-06, @@ -47785,7 +46120,6 @@ "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.2e-07, @@ -47798,7 +46132,6 @@ "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/DeepSeek-V3.2": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 3e-06, @@ -47810,7 +46143,6 @@ "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/gemma-4-31B-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3.8e-07, @@ -47826,7 +46158,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 375000, - "max_tokens": 375000, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://scx.ai/models/deepseek-v4-1-flash", @@ -47841,7 +46172,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.08e-06, "source": "https://scx.ai/models/glm-5-3", @@ -47857,7 +46187,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.413e-05, "source": "https://scx.ai/models/kimi-k3", @@ -47873,7 +46202,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.98e-06, "source": "https://scx.ai/pricing", @@ -47890,7 +46218,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.99e-06, "source": "https://scx.ai/pricing", @@ -47905,7 +46232,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 200000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -47921,7 +46247,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 1.35e-06, "output_cost_per_token": 5.4e-06, @@ -47932,56 +46257,48 @@ "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-1.5-large": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-1.5-mini": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-instruct": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama2-70b-chat": { "litellm_provider": "snowflake", "max_input_tokens": 4096, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3-70b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3-8b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.1-405b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 1.2e-06, "output_cost_per_token": 1.2e-06, @@ -47992,7 +46309,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 7.2e-07, "output_cost_per_token": 7.2e-07, @@ -48003,7 +46319,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 2.4e-07, "output_cost_per_token": 2.4e-07, @@ -48013,18 +46328,15 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.2-3b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.3-70b": { - "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 7.2e-07, @@ -48038,21 +46350,18 @@ "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/mistral-large": { "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/mistral-large2": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, @@ -48064,39 +46373,33 @@ "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/reka-core": { "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/reka-flash": { "litellm_provider": "snowflake", "max_input_tokens": 100000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-arctic": { "litellm_provider": "snowflake", "max_input_tokens": 4096, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-llama-3.1-405b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-llama-3.3-70b": { - "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 7.2e-07, @@ -48293,21 +46596,18 @@ "stability.sd3-5-large-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.08 }, "stability.sd3-large-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.08 }, "stability.stable-image-core-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.04 }, @@ -48483,21 +46783,18 @@ "stability.stable-image-core-v1:1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.04 }, "stability.stable-image-ultra-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.14 }, "stability.stable-image-ultra-v1:1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.14 }, @@ -48531,7 +46828,6 @@ "litellm_provider": "text-completion-codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "completion", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/" @@ -48541,7 +46837,6 @@ "litellm_provider": "text-completion-codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "completion", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/" @@ -48552,7 +46847,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48564,7 +46858,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48575,7 +46868,6 @@ "input_cost_per_token_batches": 6.5e-08, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_cost_per_token_batches": 0.0, @@ -48587,7 +46879,6 @@ "input_cost_per_token_batches": 1e-08, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_cost_per_token_batches": 0.0, @@ -48598,7 +46889,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -48609,7 +46899,6 @@ "input_cost_per_token_batches": 5e-08, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_cost_per_token_batches": 0.0 @@ -48619,7 +46908,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -48629,7 +46917,6 @@ "input_cost_per_token": 6.25e-09, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 3072, - "max_tokens": 3072, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48641,7 +46928,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48652,7 +46938,6 @@ "litellm_provider": "vertex_ai-text-models", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "completion", "output_cost_per_token": 2.8e-05, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -48662,7 +46947,6 @@ "litellm_provider": "vertex_ai-text-models", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "completion", "output_cost_per_token": 2.8e-05, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -48688,7 +46972,8 @@ "together-ai-8.1b-21b": { "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", - "max_tokens": 1000, + "max_output_tokens": 1000, + "max_input_tokens": 1000, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -48741,7 +47026,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "metadata": { "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" }, @@ -48757,7 +47041,7 @@ "input_cost_per_token": 1.04e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.04e-06, "source": "https://api.together.ai/v1/models", @@ -48784,7 +47068,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://api.together.ai/v1/models", @@ -48805,7 +47089,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "metadata": { "successor": "together_ai/zai-org/GLM-5.2" }, @@ -48822,7 +47106,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -48839,7 +47123,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://www.together.ai/pricing", @@ -48849,7 +47133,7 @@ "input_cost_per_token": 0.0, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.together.ai/docs/serverless-models" @@ -48858,7 +47142,7 @@ "input_cost_per_token": 1.7e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://api.together.ai/v1/models", @@ -48873,7 +47157,7 @@ "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://api.together.ai/v1/models", @@ -48884,7 +47168,7 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://www.together.ai/pricing", @@ -48894,7 +47178,7 @@ "input_cost_per_token": 3.2e-07, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.28e-06, "source": "https://api.together.ai/v1/models" @@ -48904,7 +47188,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, - "max_tokens": 1010000, + "max_output_tokens": 1010000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://api.together.ai/v1/models", @@ -48914,7 +47198,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://api.together.ai/v1/models" @@ -48925,7 +47209,7 @@ "input_cost_per_token": 1.4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://api.together.ai/v1/models", @@ -48940,7 +47224,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -48954,7 +47238,7 @@ "input_cost_per_token": 1.32e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.96e-06, "source": "https://api.together.ai/v1/models", @@ -48969,7 +47253,7 @@ "input_cost_per_token": 3.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models", @@ -48980,7 +47264,7 @@ "input_cost_per_token": 1.25e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.25e-06, "source": "https://docs.together.ai/docs/serverless-models", @@ -48993,7 +47277,7 @@ "input_cost_per_token": 3e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ @@ -49015,7 +47299,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 4.05e-06, "source": "https://api.together.ai/v1/models", @@ -49031,7 +47315,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 1048575, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.together.ai/v1/models", @@ -49048,7 +47331,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 1048575, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.together.ai/v1/models", @@ -49065,7 +47347,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 1048575, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://api.together.ai/v1/models", @@ -49150,7 +47431,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07, @@ -49169,7 +47449,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.4e-07, "output_cost_per_token_batches": 7e-08, @@ -49190,7 +47469,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, "output_cost_per_token_batches": 1.6e-06, @@ -49211,7 +47489,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -49231,7 +47508,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -49259,7 +47535,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -49277,7 +47552,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -49296,7 +47570,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -49314,7 +47587,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -49332,7 +47604,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -49371,7 +47642,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -49413,7 +47683,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -49445,7 +47714,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, @@ -49485,7 +47753,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, @@ -49524,7 +47791,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, @@ -49562,7 +47828,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -49600,7 +47865,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_adaptive_thinking": true, @@ -49636,7 +47900,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -49654,7 +47917,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", @@ -49670,7 +47932,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", @@ -49685,7 +47946,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -49703,7 +47963,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", @@ -49723,7 +47982,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", @@ -49750,7 +48008,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "supports_function_calling": true, "supports_prompt_caching": false, @@ -49767,7 +48024,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "supports_assistant_prefill": true, @@ -49797,7 +48053,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -49825,7 +48080,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -49862,7 +48116,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -49899,7 +48152,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -49939,7 +48191,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -49970,7 +48221,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", @@ -49986,7 +48236,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_batches": 9.25e-07, @@ -50007,7 +48256,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -50025,7 +48273,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -50039,7 +48286,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -50054,7 +48300,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.2e-07, "output_cost_per_token_batches": 1.1e-07, @@ -50068,7 +48313,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.6e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -50082,7 +48326,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", @@ -50095,7 +48338,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", @@ -50108,7 +48350,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -50123,7 +48364,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -50138,7 +48378,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9.7e-07, "output_cost_per_token_batches": 4.85e-07, @@ -50161,7 +48400,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 10000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_batches": 3.3e-07, @@ -50182,7 +48420,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md", @@ -50194,7 +48431,6 @@ "litellm_provider": "v0", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -50208,7 +48444,6 @@ "litellm_provider": "v0", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -50222,7 +48457,6 @@ "litellm_provider": "v0", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -50236,7 +48470,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.4e-07 }, @@ -50245,7 +48478,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -50254,7 +48486,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -50263,7 +48494,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -50274,7 +48504,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 262144, "max_output_tokens": 66536, - "max_tokens": 66536, "mode": "chat", "output_cost_per_token": 1.6e-06, "supports_function_calling": true, @@ -50285,7 +48514,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 300000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-07, "supports_vision": true, @@ -50297,7 +48525,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.4e-07, "supports_function_calling": true, @@ -50308,7 +48535,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 300000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.2e-06, "supports_vision": true, @@ -50320,7 +48546,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50331,7 +48556,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, "supports_vision": true, @@ -50346,7 +48570,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_vision": true, @@ -50361,7 +48584,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_vision": true, @@ -50376,7 +48598,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -50391,7 +48612,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -50406,7 +48626,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_vision": true, @@ -50421,7 +48640,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -50434,7 +48652,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50452,7 +48669,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50470,7 +48686,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50489,7 +48704,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -50509,7 +48723,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -50529,7 +48742,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -50549,7 +48761,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, @@ -50572,7 +48783,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": false, @@ -50593,7 +48803,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50613,7 +48822,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50630,7 +48838,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -50642,7 +48849,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -50653,7 +48859,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -50664,7 +48869,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50673,7 +48877,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.19e-06, "supports_tool_choice": true @@ -50683,7 +48886,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 9.9e-07, "supports_function_calling": true, @@ -50695,7 +48897,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07, "supports_tool_choice": true @@ -50705,7 +48906,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.5e-06, "supports_vision": true, @@ -50724,7 +48924,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "supports_vision": true, @@ -50745,7 +48944,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -50754,7 +48952,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "supports_vision": true, @@ -50766,7 +48963,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -50775,7 +48971,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -50784,7 +48979,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -50793,7 +48987,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.9e-07, "supports_tool_choice": true @@ -50803,7 +48996,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 8e-08, "supports_tool_choice": true @@ -50813,7 +49005,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_tool_choice": true @@ -50823,7 +49014,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-08, "supports_function_calling": true, @@ -50834,7 +49024,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.6e-07, "supports_vision": true, @@ -50846,7 +49035,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07 }, @@ -50855,7 +49043,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_function_calling": true, @@ -50866,7 +49053,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_vision": true, @@ -50878,7 +49064,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_function_calling": true, @@ -50889,7 +49074,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "supports_tool_choice": true @@ -50899,7 +49083,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, "supports_vision": true, @@ -50911,7 +49094,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 9e-07, "supports_function_calling": true, @@ -50922,7 +49104,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50931,7 +49112,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.8e-07, "supports_function_calling": true, @@ -50943,7 +49123,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_function_calling": true, @@ -50955,7 +49134,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true @@ -50965,7 +49143,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -50976,7 +49153,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1e-07, "supports_vision": true, @@ -50988,7 +49164,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50997,7 +49172,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -51008,7 +49182,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.9e-07 }, @@ -51017,7 +49190,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -51029,7 +49201,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 65536, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true @@ -51039,7 +49210,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_vision": true, @@ -51052,7 +49222,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 6e-06, "supports_vision": true, @@ -51065,7 +49234,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.2e-06, "supports_function_calling": true, @@ -51076,7 +49244,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -51085,7 +49252,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.9e-06 }, @@ -51094,7 +49260,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -51105,7 +49270,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06 }, @@ -51114,7 +49278,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_vision": true, @@ -51128,7 +49291,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "supports_vision": true, @@ -51143,7 +49305,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "supports_vision": true, @@ -51158,7 +49319,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "supports_vision": true, @@ -51173,7 +49333,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_vision": true, @@ -51188,7 +49347,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_vision": true, @@ -51203,7 +49361,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_vision": true, @@ -51218,7 +49375,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_vision": true, @@ -51233,7 +49389,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_function_calling": true, @@ -51247,7 +49402,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_vision": true, @@ -51260,7 +49414,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -51269,7 +49422,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -51278,7 +49430,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -51287,7 +49438,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -51296,7 +49446,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-05 }, @@ -51305,7 +49454,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 5e-06 }, @@ -51314,7 +49462,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-06 }, @@ -51323,7 +49470,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -51335,7 +49481,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -51347,7 +49492,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -51358,7 +49502,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1e-05, "supports_vision": true, @@ -51370,7 +49513,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -51381,7 +49523,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_function_calling": true @@ -51391,7 +49532,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "supports_function_calling": true, @@ -51402,7 +49542,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "supports_function_calling": true, @@ -51413,7 +49552,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -51424,7 +49562,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-06, "supports_function_calling": true, @@ -51435,7 +49572,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 96000, - "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 1.1e-06, "supports_function_calling": true, @@ -51447,7 +49583,6 @@ "input_cost_per_token": 4.5e-07, "max_input_tokens": 200000, "max_output_tokens": 200000, - "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 1.8e-06, "source": "https://vercel.com/ai-gateway/models/glm-4.6", @@ -51498,7 +49633,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -51512,7 +49646,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -51532,7 +49665,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -51561,7 +49693,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -51584,7 +49715,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51600,7 +49730,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51615,7 +49744,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, "supports_assistant_prefill": true, @@ -51629,7 +49757,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, "supports_assistant_prefill": true, @@ -51643,7 +49770,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -51657,7 +49783,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -51670,7 +49795,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51683,7 +49807,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51703,7 +49826,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51738,7 +49860,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51777,7 +49898,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51815,7 +49935,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51852,7 +49971,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51890,7 +50008,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51928,7 +50045,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -51966,7 +50082,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -52010,7 +50125,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -52048,7 +50162,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -52093,7 +50206,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52132,7 +50244,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52169,7 +50280,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52210,7 +50320,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52253,7 +50362,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52292,7 +50400,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52332,7 +50439,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -52362,7 +50468,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -52401,7 +50506,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -52440,7 +50544,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -52464,7 +50567,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52476,7 +50578,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52488,7 +50589,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52500,7 +50600,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52512,7 +50611,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -52523,7 +50621,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -52534,7 +50631,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -52548,7 +50644,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.7e-06, "output_cost_per_token_batches": 8.5e-07, @@ -52570,7 +50665,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.68e-06, "output_cost_per_token_batches": 8.4e-07, @@ -52591,7 +50685,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 65336, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "output_cost_per_token_batches": 2.7e-06, @@ -52614,7 +50707,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, @@ -52673,7 +50765,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -52708,7 +50799,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -52741,7 +50831,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -52769,7 +50858,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -52799,7 +50887,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -52839,7 +50926,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -52898,7 +50984,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -52959,7 +51044,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -53016,7 +51100,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -53030,7 +51113,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -53041,7 +51123,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -53052,7 +51133,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -53063,7 +51143,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -53074,7 +51153,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -53104,7 +51182,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "audio_speech", "output_cost_per_image": 0.04, "output_cost_per_token": 0, @@ -53140,7 +51217,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "audio_speech", "output_cost_per_image": 0.08, "output_cost_per_token": 0, @@ -53177,7 +51253,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.6e-05, "source": "https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.2-90b-vision-instruct-maas", @@ -53190,7 +51265,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.2-90b-vision-instruct-maas", @@ -53203,7 +51277,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "metadata": { "notes": "VertexAI states that The Llama 3.1 API service for llama-3.1-70b-instruct-maas and llama-3.1-8b-instruct-maas are in public preview and at no cost." }, @@ -53219,7 +51292,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "metadata": { "notes": "VertexAI states that The Llama 3.2 API service is at no cost during public preview, and will be priced as per dollar-per-1M-tokens at GA." }, @@ -53236,7 +51308,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, "output_cost_per_token_batches": 5.75e-07, @@ -53258,7 +51329,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, "output_cost_per_token_batches": 5.75e-07, @@ -53280,7 +51350,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, - "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, "output_cost_per_token_batches": 3.5e-07, @@ -53302,7 +51371,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, - "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, "output_cost_per_token_batches": 3.5e-07, @@ -53323,7 +51391,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -53334,7 +51401,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -53345,7 +51411,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -53358,7 +51423,6 @@ "litellm_provider": "vertex_ai-minimax_models", "max_input_tokens": 196608, "max_output_tokens": 196608, - "max_tokens": 196608, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53372,7 +51436,6 @@ "litellm_provider": "vertex_ai-moonshot_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53387,7 +51450,6 @@ "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53405,7 +51467,6 @@ "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53422,7 +51483,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53434,7 +51494,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53446,7 +51505,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53458,7 +51516,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53470,7 +51527,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53481,7 +51537,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53492,7 +51547,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53503,7 +51557,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53514,7 +51567,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -53525,7 +51577,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_function_calling": true, @@ -53536,7 +51587,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -53549,7 +51599,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -53583,7 +51632,6 @@ "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 262144, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/maas/google/gemma-4-26b-a4b-it", @@ -53600,7 +51648,6 @@ "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.6e-07, "output_cost_per_token_batches": 1.8e-07, @@ -53613,7 +51660,6 @@ "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53628,7 +51674,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53645,7 +51690,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53665,7 +51709,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, - "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -53685,7 +51728,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, - "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -53706,7 +51748,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 200000, "max_output_tokens": 200000, - "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -53726,7 +51767,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -53745,7 +51785,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 8.8e-07, "output_cost_per_token_batches": 4.4e-07, @@ -53765,7 +51804,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_batches": 9e-07, @@ -53782,7 +51820,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53798,7 +51835,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53811,7 +51847,6 @@ "vertex_ai/veo-3.1-generate-preview": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -53826,7 +51861,6 @@ "vertex_ai/veo-3.1-fast-generate-preview": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -53843,7 +51877,6 @@ "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -53859,7 +51892,6 @@ "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -53875,7 +51907,6 @@ "vertex_ai/veo-3.1-lite-generate-001": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.05, "output_cost_per_second_1080p": 0.08, @@ -53893,7 +51924,6 @@ "litellm_provider": "voyage", "max_input_tokens": 16000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53902,7 +51932,6 @@ "litellm_provider": "voyage", "max_input_tokens": 8000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53911,7 +51940,6 @@ "litellm_provider": "voyage", "max_input_tokens": 8000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53921,7 +51949,6 @@ "litellm_provider": "voyage", "max_input_tokens": 4000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53931,7 +51958,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53940,7 +51966,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53949,7 +51974,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53959,7 +51983,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53968,7 +51991,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "voyage", "max_input_tokens": 4000, - "max_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -53976,7 +51998,6 @@ "input_cost_per_token": 6e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -53984,7 +52005,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -53994,7 +52014,6 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54002,7 +52021,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54010,7 +52028,6 @@ "input_cost_per_token": 6e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54018,7 +52035,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54026,7 +52042,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54034,7 +52049,6 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54042,7 +52056,6 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", "max_input_tokens": 120000, - "max_tokens": 120000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54050,7 +52063,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54058,7 +52070,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54066,7 +52077,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -54076,7 +52086,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54084,7 +52093,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "voyage", "max_input_tokens": 4096, - "max_tokens": 4096, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54092,7 +52100,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "voyage", "max_input_tokens": 4000, - "max_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54100,13 +52107,11 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, "wandb/openai/gpt-oss-120b": { "supports_reasoning": true, - "max_tokens": 131072, "max_input_tokens": 131000, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -54117,7 +52122,6 @@ }, "wandb/openai/gpt-oss-20b": { "supports_reasoning": true, - "max_tokens": 131072, "max_input_tokens": 131000, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -54127,7 +52131,6 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/moonshotai/Kimi-K2.5": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "cache_read_input_token_cost": 1e-07, @@ -54142,7 +52145,6 @@ "supports_vision": true }, "wandb/meta-llama/Llama-3.1-8B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 131000, "max_output_tokens": 128000, "input_cost_per_token": 2.2e-07, @@ -54153,7 +52155,6 @@ }, "wandb/deepseek-ai/DeepSeek-V3.1": { "supports_reasoning": true, - "max_tokens": 128000, "max_input_tokens": 161000, "max_output_tokens": 128000, "input_cost_per_token": 5.5e-07, @@ -54163,7 +52164,6 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 7.1e-07, @@ -54177,7 +52177,6 @@ "litellm_provider": "watsonx", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 2e-07, "supports_audio_input": false, @@ -54195,7 +52194,6 @@ "litellm_provider": "watsonx", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_audio_input": false, @@ -54209,7 +52207,6 @@ "supports_vision": false }, "watsonx/bigscience/mt0-xxl-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.908e-06, @@ -54222,7 +52219,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/bigscience/mt0-xxl": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.908e-06, @@ -54235,7 +52231,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/core42/jais-13b-chat": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 0.0005, @@ -54247,7 +52242,6 @@ "supports_vision": false }, "watsonx/google/flan-t5-xl-3b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 6e-07, @@ -54259,7 +52253,6 @@ "supports_vision": false }, "watsonx/ibm/granite-13b-chat-v2": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 6e-07, @@ -54271,7 +52264,6 @@ "supports_vision": false }, "watsonx/ibm/granite-13b-instruct-v2": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 6e-07, @@ -54283,7 +52275,6 @@ "supports_vision": false }, "watsonx/ibm/granite-3-3-8b-instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -54295,7 +52286,6 @@ "supports_vision": false }, "watsonx/ibm/granite-4-h-small": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6.36e-08, @@ -54308,7 +52298,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/ibm/granite-guardian-3-2-2b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -54320,7 +52309,6 @@ "supports_vision": false }, "watsonx/ibm/granite-guardian-3-3-8b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -54332,7 +52320,6 @@ "supports_vision": false }, "watsonx/ibm/granite-ttm-1024-96-r2": { - "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, "input_cost_per_token": 3.8e-07, @@ -54344,7 +52331,6 @@ "supports_vision": false }, "watsonx/ibm/granite-ttm-1536-96-r2": { - "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, "input_cost_per_token": 3.8e-07, @@ -54356,7 +52342,6 @@ "supports_vision": false }, "watsonx/ibm/granite-ttm-512-96-r2": { - "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, "input_cost_per_token": 3.8e-07, @@ -54368,7 +52353,6 @@ "supports_vision": false }, "watsonx/ibm/granite-vision-3-2-2b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -54380,7 +52364,6 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-2-11b-vision-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3.5e-07, @@ -54392,7 +52375,6 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-2-1b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1e-07, @@ -54404,7 +52386,6 @@ "supports_vision": false }, "watsonx/meta-llama/llama-3-2-3b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.5e-07, @@ -54416,7 +52397,6 @@ "supports_vision": false }, "watsonx/meta-llama/llama-3-2-90b-vision-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-06, @@ -54428,7 +52408,6 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-3-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 7.526e-07, @@ -54441,7 +52420,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 3.71e-07, @@ -54454,7 +52432,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 3.71e-07, @@ -54467,7 +52444,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-guard-3-11b-vision": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3.5e-07, @@ -54479,7 +52455,6 @@ "supports_vision": true }, "watsonx/mistralai/mistral-medium-2505": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, @@ -54491,7 +52466,6 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-2503": { - "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, "input_cost_per_token": 1e-07, @@ -54503,7 +52477,6 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-3-1-24b-instruct-2503": { - "max_tokens": 16384, "max_input_tokens": 131072, "max_output_tokens": 16384, "input_cost_per_token": 1.06e-07, @@ -54516,7 +52489,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/mistralai/pixtral-12b-2409": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3.5e-07, @@ -54528,7 +52500,6 @@ "supports_vision": true }, "watsonx/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.59e-07, @@ -54541,7 +52512,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/sdaia/allam-1-13b-instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1.8e-06, @@ -54578,7 +52548,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -54603,7 +52572,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -54627,7 +52595,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -54653,7 +52620,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -54681,7 +52647,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -54709,7 +52674,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -54733,7 +52697,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54755,7 +52718,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54777,7 +52739,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54799,7 +52760,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54821,7 +52781,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54843,7 +52802,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "output_cost_per_token_batches": 1.1e-06, @@ -54866,7 +52824,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_batches": 1.6e-06, "output_cost_per_token_priority": 5.6e-06, @@ -54889,7 +52846,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -55487,7 +53443,6 @@ } }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-480b-a35b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 4.5e-07, @@ -55497,7 +53452,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/flux-kontext-pro": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 4e-08, @@ -55506,7 +53460,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/SSD-1B": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -55515,7 +53468,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/chronos-hermes-13b-v2": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55524,7 +53476,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-13b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55533,7 +53484,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-13b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55542,7 +53492,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-13b-python": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55551,7 +53500,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-34b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55560,7 +53508,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-34b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55569,7 +53516,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-34b-python": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55578,7 +53524,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-70b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -55587,7 +53532,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-70b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -55596,7 +53540,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-70b-python": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -55605,7 +53548,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-7b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55614,7 +53556,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-7b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55623,7 +53564,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-7b-python": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55632,7 +53572,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-qwen-1p5-7b": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 2e-07, @@ -55641,7 +53580,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/codegemma-2b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -55650,7 +53588,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/codegemma-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -55659,7 +53596,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-671b-v2-p1": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1.2e-06, @@ -55668,7 +53604,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-llama-3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -55677,7 +53612,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-llama-70b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55686,7 +53620,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-llama-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55695,7 +53628,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-qwen-14b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55704,7 +53636,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-qwen-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55713,7 +53644,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-kontext-max": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 8e-08, @@ -55722,7 +53652,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/dbrx-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.2e-06, @@ -55731,7 +53660,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-1b-base": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 1e-07, @@ -55740,7 +53668,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-33b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55749,7 +53676,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-7b-base": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55758,7 +53684,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-7b-base-v1p5": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55767,7 +53692,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-7b-instruct-v1p5": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55776,7 +53700,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-v2-lite-base": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -55785,7 +53708,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-v2-lite-instruct": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -55794,7 +53716,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-prover-v2": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1.2e-06, @@ -55803,7 +53724,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-0528-distill-qwen3-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55812,7 +53732,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-llama-70b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55821,7 +53740,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-llama-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55830,7 +53748,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-14b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55839,7 +53756,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-1p5b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -55848,7 +53764,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55857,7 +53772,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-7b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55866,7 +53780,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-v2-lite-chat": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -55875,7 +53788,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-v2p5": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.2e-06, @@ -55884,7 +53796,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/devstral-small-2505": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55893,7 +53804,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dobby-mini-unhinged-plus-llama-3-1-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55902,7 +53812,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dobby-unhinged-llama-3-3-70b-new": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55911,7 +53820,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dolphin-2-9-2-qwen2-72b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55920,7 +53828,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dolphin-2p6-mixtral-8x7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -55929,7 +53836,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ernie-4p5-21b-a3b-pt": { - "max_tokens": 4096, "max_input_tokens": 131072, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -55938,7 +53844,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ernie-4p5-300b-a47b-pt": { - "max_tokens": 4096, "max_input_tokens": 131072, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -55947,7 +53852,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/fare-20b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55956,7 +53860,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/firefunction-v1": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -55965,7 +53868,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/firellava-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55974,7 +53876,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/firesearch-ocr-v6": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -55983,7 +53884,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/fireworks-asr-large": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.0, @@ -55992,7 +53892,6 @@ "mode": "audio_transcription" }, "fireworks_ai/accounts/fireworks/models/fireworks-asr-v2": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.0, @@ -56001,7 +53900,6 @@ "mode": "audio_transcription" }, "fireworks_ai/accounts/fireworks/models/flux-1-dev": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56010,7 +53908,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-1-dev-controlnet-union": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-09, @@ -56019,7 +53916,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-1-dev-fp8": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 5e-10, @@ -56028,7 +53924,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/flux-1-schnell": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56037,7 +53932,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-1-schnell-fp8": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 3.5e-10, @@ -56046,7 +53940,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/gemma-2b-it": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -56055,7 +53948,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma-3-27b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56064,7 +53956,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56073,7 +53964,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma-7b-it": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56082,7 +53972,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma2-9b-it": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56091,7 +53980,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/glm-4p5v": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.2e-06, @@ -56101,7 +53989,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/gpt-oss-safeguard-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.2e-06, @@ -56110,7 +53997,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gpt-oss-safeguard-20b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-07, @@ -56119,7 +54005,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/hermes-2-pro-mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56128,7 +54013,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/internvl3-38b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56137,7 +54021,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/internvl3-78b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56146,7 +54029,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/internvl3-8b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -56155,7 +54037,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/japanese-stable-diffusion-xl": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -56164,7 +54045,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/kat-coder": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-07, @@ -56173,7 +54053,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/kat-dev-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56182,7 +54061,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/kat-dev-72b-exp": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56191,7 +54069,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-guard-2-8b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56200,7 +54077,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-guard-3-1b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56209,7 +54085,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-guard-3-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56218,7 +54093,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56227,7 +54101,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-13b-chat": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56236,7 +54109,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-70b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56245,7 +54117,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-70b-chat": { - "max_tokens": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048, "input_cost_per_token": 9e-07, @@ -56254,7 +54125,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-7b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56263,7 +54133,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-7b-chat": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56272,7 +54141,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 9e-07, @@ -56281,7 +54149,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct-hf": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 9e-07, @@ -56290,7 +54157,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-8b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56299,7 +54165,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-8b-instruct-hf": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56308,7 +54173,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct-long": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56317,7 +54181,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56326,7 +54189,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct-1b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56335,7 +54197,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-nemotron-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56344,7 +54205,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p2-1b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56353,7 +54213,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p2-3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56362,7 +54221,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56371,7 +54229,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llamaguard-7b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56380,7 +54237,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llava-yi-34b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -56389,7 +54245,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/minimax-m1-80k": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56398,7 +54253,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/minimax-m2": { - "max_tokens": 4096, "max_input_tokens": 196608, "max_output_tokens": 4096, "input_cost_per_token": 3e-07, @@ -56407,7 +54261,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ministral-3-14b-instruct-2512": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 2e-07, @@ -56416,7 +54269,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ministral-3-3b-instruct-2512": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 1e-07, @@ -56425,7 +54277,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ministral-3-8b-instruct-2512": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 2e-07, @@ -56434,7 +54285,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56443,7 +54293,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-instruct-4k": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56452,7 +54301,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-instruct-v0p2": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56461,7 +54309,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-instruct-v3": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56470,7 +54317,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-v0p2": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56479,7 +54325,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-large-3-fp8": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 1.2e-06, @@ -56488,7 +54333,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-nemo-base-2407": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -56497,7 +54341,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-nemo-instruct-2407": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -56506,7 +54349,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-small-24b-instruct-2501": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56515,7 +54357,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 1.2e-06, @@ -56524,7 +54365,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 1.2e-06, @@ -56533,7 +54373,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56542,7 +54381,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56551,7 +54389,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x7b-instruct-hf": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56560,7 +54397,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mythomax-l2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56569,7 +54405,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nemotron-nano-v2-12b-vl": { - "max_tokens": 4096, "max_input_tokens": 131072, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56578,7 +54413,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-capybara-7b-v1p9": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56587,7 +54421,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-2-mixtral-8x7b-dpo": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56596,7 +54429,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-2-yi-34b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -56605,7 +54437,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-llama2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56614,7 +54445,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-llama2-70b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -56623,7 +54453,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-llama2-7b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56632,7 +54461,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nvidia-nemotron-nano-12b-v2": { - "max_tokens": 131072, "max_input_tokens": 128000, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56641,7 +54469,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nvidia-nemotron-nano-9b-v2": { - "max_tokens": 131072, "max_input_tokens": 128000, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56650,7 +54477,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openchat-3p5-0106-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56659,7 +54485,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openhermes-2-mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56668,7 +54493,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openhermes-2p5-mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56677,7 +54501,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openorca-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56686,7 +54509,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phi-2-3b": { - "max_tokens": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048, "input_cost_per_token": 1e-07, @@ -56695,7 +54517,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phi-3-mini-128k-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56704,7 +54525,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phi-3-vision-128k-instruct": { - "max_tokens": 32064, "max_input_tokens": 32064, "max_output_tokens": 32064, "input_cost_per_token": 2e-07, @@ -56713,7 +54533,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phind-code-llama-34b-python-v1": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56722,7 +54541,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phind-code-llama-34b-v1": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56731,7 +54549,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phind-code-llama-34b-v2": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56740,7 +54557,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/playground-v2-1024px-aesthetic": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -56749,7 +54565,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/playground-v2-5-1024px-aesthetic": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -56758,7 +54573,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/pythia-12b": { - "max_tokens": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048, "input_cost_per_token": 2e-07, @@ -56767,7 +54581,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen-qwq-32b-preview": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56776,7 +54589,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen-v2p5-14b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56785,7 +54597,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen-v2p5-7b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56794,7 +54605,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen1p5-72b-chat": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56803,7 +54613,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56812,7 +54621,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-vl-2b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56821,7 +54629,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-vl-72b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56830,7 +54637,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-vl-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56839,7 +54645,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-0p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56848,7 +54653,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-14b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56857,7 +54661,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-1p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56866,7 +54669,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56875,7 +54677,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-32b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56884,7 +54685,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-72b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56893,7 +54693,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-72b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56902,7 +54701,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56911,7 +54709,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-0p5b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56920,7 +54717,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-0p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56929,7 +54725,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-14b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56938,7 +54733,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-14b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56947,7 +54741,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-1p5b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56956,7 +54749,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-1p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56965,7 +54757,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56974,7 +54765,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct-128k": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56983,7 +54773,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct-32k-rope": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56992,7 +54781,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct-64k": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 9e-07, @@ -57001,7 +54789,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-3b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -57010,7 +54797,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-3b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -57019,7 +54805,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57028,7 +54813,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57037,7 +54821,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-math-72b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57046,7 +54829,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-32b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 9e-07, @@ -57055,7 +54837,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-3b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -57064,7 +54845,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-72b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 9e-07, @@ -57073,7 +54853,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-7b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -57082,7 +54861,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-0p6b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-07, @@ -57091,7 +54869,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-14b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57100,7 +54877,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -57109,7 +54885,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b-fp8-draft": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-07, @@ -57118,7 +54893,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b-fp8-draft-131072": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -57127,7 +54901,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b-fp8-draft-40960": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-07, @@ -57136,7 +54909,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.2e-07, @@ -57145,7 +54917,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b-instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57154,7 +54925,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b-thinking-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57163,7 +54933,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-30b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.5e-07, @@ -57172,7 +54941,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-30b-a3b-instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 5e-07, @@ -57181,7 +54949,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-30b-a3b-thinking-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-07, @@ -57190,7 +54957,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -57200,7 +54966,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/qwen3-4b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57209,7 +54974,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-4b-instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2e-07, @@ -57218,7 +54982,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-8b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57228,7 +54991,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-30b-a3b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -57237,7 +54999,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-480b-instruct-bf16": { - "max_tokens": 4096, "max_input_tokens": 262144, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57246,7 +55007,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-0p6b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 0.0, @@ -57255,7 +55015,6 @@ "mode": "embedding" }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-4b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 0.0, @@ -57264,7 +55023,6 @@ "mode": "embedding" }, "fireworks_ai/accounts/fireworks/models/qwen3-next-80b-a3b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57273,7 +55031,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-next-80b-a3b-thinking": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57282,7 +55039,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-reranker-0p6b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 0.0, @@ -57291,7 +55047,6 @@ "mode": "rerank" }, "fireworks_ai/accounts/fireworks/models/qwen3-reranker-4b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 0.0, @@ -57300,7 +55055,6 @@ "mode": "rerank" }, "fireworks_ai/accounts/fireworks/models/qwen3-reranker-8b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57310,7 +55064,6 @@ "source": "https://api.fireworks.ai/v1/serverless/models" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-235b-a22b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57319,7 +55072,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-235b-a22b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57328,7 +55080,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-30b-a3b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -57337,7 +55088,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-30b-a3b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -57346,7 +55096,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-32b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57355,7 +55104,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-8b-instruct": { - "max_tokens": 4096, "max_input_tokens": 262144, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -57369,7 +55117,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -57380,7 +55127,6 @@ "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/qwq-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -57389,7 +55135,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/rolm-ocr": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -57398,7 +55143,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/snorkel-mistral-7b-pairrm-dpo": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57407,7 +55151,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/stable-diffusion-xl-1024-v1-0": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -57416,7 +55159,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/stablecode-3b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -57425,7 +55167,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder-16b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -57434,7 +55175,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -57443,7 +55183,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder2-15b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -57452,7 +55191,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder2-3b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 1e-07, @@ -57461,7 +55199,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder2-7b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -57470,7 +55207,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/toppy-m-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57479,7 +55215,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-34b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57488,7 +55223,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-34b-200k-capybara": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 9e-07, @@ -57497,7 +55231,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-34b-chat": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57506,7 +55239,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-6b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -57515,7 +55247,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/zephyr-7b-beta": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57529,7 +55260,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -57546,7 +55276,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -57563,7 +55292,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -57578,7 +55306,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -57590,7 +55317,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -57602,7 +55328,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.25e-06, "supports_function_calling": true @@ -57618,7 +55343,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -57629,7 +55353,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true @@ -57645,7 +55368,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 5e-07, "supports_function_calling": true, @@ -57657,7 +55379,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.5e-06, "supports_reasoning": true, @@ -57670,7 +55391,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.5e-07, "supports_function_calling": true, @@ -57681,7 +55401,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "supports_vision": true, @@ -57699,7 +55418,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 9e-07, "supports_function_calling": true @@ -57711,7 +55429,6 @@ "output_cost_per_token": 4e-07, "max_input_tokens": 163840, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57728,7 +55445,6 @@ "output_cost_per_token": 1.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57744,7 +55460,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57761,7 +55476,6 @@ "output_cost_per_token": 3.3e-07, "max_input_tokens": 262144, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57778,7 +55492,6 @@ "output_cost_per_token": 1.38e-07, "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_vision": true, "supports_system_messages": true }, @@ -57789,7 +55502,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, "max_output_tokens": 100352, - "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57806,7 +55518,6 @@ "output_cost_per_token": 1.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57823,7 +55534,6 @@ "output_cost_per_token": 2e-08, "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_vision": true, "supports_system_messages": true }, @@ -57834,7 +55544,6 @@ "output_cost_per_token": 4.1e-07, "max_input_tokens": 163840, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57849,7 +55558,6 @@ "output_cost_per_token": 3.95e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_vision": true, "supports_system_messages": true, "supports_reasoning": true, @@ -57863,7 +55571,6 @@ "output_cost_per_token": 9e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57881,7 +55588,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57898,7 +55604,6 @@ "output_cost_per_token": 1.2e-06, "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57914,7 +55619,6 @@ "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57929,7 +55633,6 @@ "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57944,7 +55647,6 @@ "output_cost_per_token": 3e-08, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_vision": true, "supports_system_messages": true, @@ -57957,7 +55659,6 @@ "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57974,7 +55675,6 @@ "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57989,7 +55689,6 @@ "output_cost_per_token": 8.45e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58003,7 +55702,6 @@ "output_cost_per_token": 6e-07, "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_tool_choice": true, "supports_vision": true, "supports_system_messages": true, @@ -58016,7 +55714,6 @@ "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58033,7 +55730,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, "max_output_tokens": 100352, - "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58047,7 +55743,6 @@ "output_cost_per_token": 1.55e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58061,7 +55756,6 @@ "output_cost_per_token": 2.7e-07, "max_input_tokens": 160000, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58075,7 +55769,6 @@ "output_cost_per_token": 2.5e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58091,7 +55784,6 @@ "output_cost_per_token": 2.3e-06, "max_input_tokens": 131072, "max_output_tokens": 100352, - "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58105,7 +55797,6 @@ "output_cost_per_token": 1.12e-06, "max_input_tokens": 163840, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58121,7 +55812,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 131072, "max_output_tokens": 98304, - "max_tokens": 98304, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58137,7 +55827,6 @@ "output_cost_per_token": 3e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58151,7 +55840,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_system_messages": true, "supports_response_schema": true }, @@ -58162,7 +55850,6 @@ "output_cost_per_token": 1e-07, "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_vision": true, "supports_system_messages": true, @@ -58175,7 +55862,6 @@ "output_cost_per_token": 1.8e-06, "max_input_tokens": 65536, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58193,7 +55879,6 @@ "output_cost_per_token": 1.5e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_tool_choice": true, "supports_vision": false, "supports_system_messages": true, @@ -58207,7 +55892,6 @@ "output_cost_per_token": 5.8e-07, "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58221,7 +55905,6 @@ "output_cost_per_token": 1.5e-07, "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, @@ -58234,7 +55917,6 @@ "output_cost_per_token": 4e-07, "max_input_tokens": 12288, "max_output_tokens": 12288, - "max_tokens": 12288, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58248,7 +55930,6 @@ "output_cost_per_token": 4e-07, "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58262,7 +55943,6 @@ "output_cost_per_token": 1.7e-07, "max_input_tokens": 60288, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58274,7 +55954,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 1000000, "max_output_tokens": 40000, - "max_tokens": 40000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58289,7 +55968,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58306,7 +55984,6 @@ "output_cost_per_token": 3e-07, "max_input_tokens": 64000, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, @@ -58319,7 +55996,6 @@ "output_cost_per_token": 4e-08, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_system_messages": true, "supports_response_schema": true }, @@ -58330,7 +56006,6 @@ "output_cost_per_token": 6.2e-07, "max_input_tokens": 65535, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_system_messages": true, "supports_response_schema": true }, @@ -58341,7 +56016,6 @@ "output_cost_per_token": 9e-08, "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_system_messages": true, "supports_reasoning": true }, @@ -58352,7 +56026,6 @@ "output_cost_per_token": 8e-07, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, @@ -58365,7 +56038,6 @@ "output_cost_per_token": 7.4e-07, "max_input_tokens": 8192, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58377,7 +56049,6 @@ "output_cost_per_token": 8e-07, "max_input_tokens": 40960, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true @@ -58389,7 +56060,6 @@ "output_cost_per_token": 8.5e-07, "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_vision": true, "supports_system_messages": true, "supports_response_schema": true @@ -58401,7 +56071,6 @@ "output_cost_per_token": 5.9e-07, "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_vision": true, "supports_system_messages": true }, @@ -58412,7 +56081,6 @@ "output_cost_per_token": 1.4e-07, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58424,7 +56092,6 @@ "output_cost_per_token": 8e-07, "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_vision": true, "supports_system_messages": true }, @@ -58435,7 +56102,6 @@ "output_cost_per_token": 1.48e-06, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58448,7 +56114,6 @@ "output_cost_per_token": 2.8e-07, "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_system_messages": true, "supports_reasoning": true }, @@ -58459,7 +56124,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58471,7 +56135,6 @@ "output_cost_per_token": 7e-08, "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-424b-a47b": { @@ -58481,7 +56144,6 @@ "output_cost_per_token": 1.25e-06, "max_input_tokens": 123000, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_vision": true, "supports_system_messages": true, "supports_reasoning": true @@ -58493,7 +56155,6 @@ "output_cost_per_token": 1.1e-06, "max_input_tokens": 123000, "max_output_tokens": 12000, - "max_tokens": 12000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58505,7 +56166,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 160000, "max_output_tokens": 160000, - "max_tokens": 160000, "supports_system_messages": true }, "novita/qwen/qwen3-32b-fp8": { @@ -58515,7 +56175,6 @@ "output_cost_per_token": 4.5e-07, "max_input_tokens": 40960, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true }, @@ -58526,7 +56185,6 @@ "output_cost_per_token": 4.5e-07, "max_input_tokens": 40960, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true, "supports_function_calling": true, @@ -58539,7 +56197,6 @@ "output_cost_per_token": 2e-07, "max_input_tokens": 98304, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_vision": true, "supports_system_messages": true }, @@ -58550,7 +56207,6 @@ "output_cost_per_token": 1.3e-06, "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58563,7 +56219,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58578,7 +56233,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 8192, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58591,7 +56245,6 @@ "output_cost_per_token": 9e-08, "max_input_tokens": 4096, "max_output_tokens": 3200, - "max_tokens": 3200, "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-28b-a3b-thinking": { @@ -58601,7 +56254,6 @@ "output_cost_per_token": 3.9e-07, "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58617,7 +56269,6 @@ "output_cost_per_token": 5e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58632,7 +56283,6 @@ "output_cost_per_token": 8.5e-07, "max_input_tokens": 131072, "max_output_tokens": 98304, - "max_tokens": 98304, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58648,7 +56298,6 @@ "output_cost_per_token": 7e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58663,7 +56312,6 @@ "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58678,7 +56326,6 @@ "output_cost_per_token": 9.7e-07, "max_input_tokens": 65536, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58695,7 +56342,6 @@ "output_cost_per_token": 9.7e-07, "max_input_tokens": 65536, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58712,7 +56358,6 @@ "output_cost_per_token": 7.5e-07, "max_input_tokens": 16384, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-28b-a3b": { @@ -58722,7 +56367,6 @@ "output_cost_per_token": 5.6e-07, "max_input_tokens": 30000, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58737,7 +56381,6 @@ "output_cost_per_token": 2.8e-07, "max_input_tokens": 120000, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58750,7 +56393,6 @@ "output_cost_per_token": 1.38e-07, "max_input_tokens": 128000, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true }, @@ -58761,7 +56403,6 @@ "output_cost_per_token": 3e-08, "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_system_messages": true, "supports_reasoning": true, "supports_function_calling": true, @@ -58774,7 +56415,6 @@ "output_cost_per_token": 7e-08, "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58788,7 +56428,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 32768, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58801,7 +56440,6 @@ "output_cost_per_token": 1.48e-06, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58813,8 +56451,7 @@ "input_cost_per_token": 7e-08, "output_cost_per_token": 0, "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "novita/qwen/qwen3-embedding-8b": { "litellm_provider": "novita", @@ -58822,8 +56459,7 @@ "input_cost_per_token": 7e-08, "output_cost_per_token": 0, "max_input_tokens": 32768, - "max_output_tokens": 4096, - "max_tokens": 4096 + "max_output_tokens": 4096 }, "novita/baai/bge-m3": { "litellm_provider": "novita", @@ -58831,8 +56467,7 @@ "input_cost_per_token": 1e-08, "output_cost_per_token": 1e-08, "max_input_tokens": 8192, - "max_output_tokens": 96000, - "max_tokens": 96000 + "max_output_tokens": 96000 }, "novita/qwen/qwen3-reranker-8b": { "litellm_provider": "novita", @@ -58840,8 +56475,7 @@ "input_cost_per_token": 5e-08, "output_cost_per_token": 5e-08, "max_input_tokens": 32768, - "max_output_tokens": 4096, - "max_tokens": 4096 + "max_output_tokens": 4096 }, "novita/baai/bge-reranker-v2-m3": { "litellm_provider": "novita", @@ -58849,11 +56483,9 @@ "input_cost_per_token": 1e-08, "output_cost_per_token": 1e-08, "max_input_tokens": 8000, - "max_output_tokens": 8000, - "max_tokens": 8000 + "max_output_tokens": 8000 }, "llamagate/llama-3.1-8b": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 3e-08, @@ -58864,7 +56496,6 @@ "supports_response_schema": true }, "llamagate/llama-3.2-3b": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 4e-08, @@ -58875,7 +56506,6 @@ "supports_response_schema": true }, "llamagate/mistral-7b-v0.3": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -58886,7 +56516,6 @@ "supports_response_schema": true }, "llamagate/qwen3-8b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 4e-08, @@ -58897,7 +56526,6 @@ "supports_response_schema": true }, "llamagate/dolphin3-8b": { - "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "input_cost_per_token": 8e-08, @@ -58908,7 +56536,6 @@ "supports_response_schema": true }, "llamagate/deepseek-r1-8b": { - "max_tokens": 16384, "max_input_tokens": 65536, "max_output_tokens": 16384, "input_cost_per_token": 1e-07, @@ -58920,7 +56547,6 @@ "supports_reasoning": true }, "llamagate/deepseek-r1-7b-qwen": { - "max_tokens": 16384, "max_input_tokens": 131072, "max_output_tokens": 16384, "input_cost_per_token": 8e-08, @@ -58932,7 +56558,6 @@ "supports_reasoning": true }, "llamagate/openthinker-7b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 8e-08, @@ -58944,7 +56569,6 @@ "supports_reasoning": true }, "llamagate/qwen2.5-coder-7b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 6e-08, @@ -58955,7 +56579,6 @@ "supports_response_schema": true }, "llamagate/deepseek-coder-6.7b": { - "max_tokens": 4096, "max_input_tokens": 16384, "max_output_tokens": 4096, "input_cost_per_token": 6e-08, @@ -58966,7 +56589,6 @@ "supports_response_schema": true }, "llamagate/codellama-7b": { - "max_tokens": 4096, "max_input_tokens": 16384, "max_output_tokens": 4096, "input_cost_per_token": 6e-08, @@ -58977,7 +56599,6 @@ "supports_response_schema": true }, "llamagate/qwen3-vl-8b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 1.5e-07, @@ -58989,7 +56610,6 @@ "supports_vision": true }, "llamagate/llava-7b": { - "max_tokens": 2048, "max_input_tokens": 4096, "max_output_tokens": 2048, "input_cost_per_token": 1e-07, @@ -59000,7 +56620,6 @@ "supports_vision": true }, "llamagate/gemma3-4b": { - "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "input_cost_per_token": 3e-08, @@ -59012,7 +56631,6 @@ "supports_vision": true }, "llamagate/nomic-embed-text": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 2e-08, "output_cost_per_token": 0, @@ -59020,7 +56638,6 @@ "mode": "embedding" }, "llamagate/qwen3-embedding-8b": { - "max_tokens": 40960, "max_input_tokens": 40960, "input_cost_per_token": 2e-08, "output_cost_per_token": 0, @@ -59028,7 +56645,6 @@ "mode": "embedding" }, "libertai/hermes-3-8b-tee": { - "max_tokens": 16000, "max_input_tokens": 16000, "max_output_tokens": 16000, "input_cost_per_token": 1.5e-07, @@ -59042,7 +56658,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/gemma-4-31b-it": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59056,7 +56671,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/gemma-4-31b-it-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59071,7 +56685,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-27b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59085,7 +56698,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-27b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59100,7 +56712,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-35b-a3b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59114,7 +56725,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-35b-a3b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59129,7 +56739,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.5-122b-a10b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -59143,7 +56752,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.5-122b-a10b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -59158,7 +56766,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/deepseek-v4-flash": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 2.5e-07, @@ -59172,7 +56779,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/deepseek-v4-flash-thinking": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 2.5e-07, @@ -59187,7 +56793,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/bge-m3": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -59203,7 +56808,6 @@ "litellm_provider": "sarvam", "max_input_tokens": 8192, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0, "supports_reasoning": true @@ -59279,7 +56883,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -59307,7 +56910,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -59339,7 +56941,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -59449,7 +57050,6 @@ "litellm_provider": "gemini", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_token": 0.0, @@ -59470,7 +57070,6 @@ "litellm_provider": "gemini", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_token": 0.0, @@ -59493,7 +57092,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59519,7 +57117,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59546,7 +57143,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59581,7 +57177,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59616,7 +57211,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59650,7 +57244,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59683,7 +57276,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59711,7 +57303,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59740,7 +57331,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59778,7 +57368,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59815,7 +57404,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -59842,7 +57430,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 9e-06, "output_cost_per_token": 9e-06, @@ -59867,7 +57454,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 6e-06, "output_cost_per_token": 6e-06, @@ -59887,7 +57473,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, @@ -59907,7 +57492,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -59966,7 +57550,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -60026,7 +57609,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -60088,7 +57670,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -60149,7 +57730,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -60200,7 +57780,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -60239,7 +57818,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -60281,7 +57859,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -60307,7 +57884,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token_batches": 1.5e-07, "output_cost_per_token_priority": 5.25e-07, @@ -60333,7 +57909,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -60357,7 +57932,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 1e-07, "output_cost_per_token_priority": 3.5e-07, @@ -60387,7 +57961,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60431,7 +58004,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60466,7 +58038,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-cyber.html", "use_openai_responses_path": true, @@ -60502,7 +58073,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60542,7 +58112,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60583,7 +58152,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html", "supported_modalities": [ @@ -60620,7 +58188,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html", "supported_modalities": [ @@ -60657,7 +58224,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", "supported_modalities": [ @@ -60694,7 +58260,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", "supported_modalities": [ @@ -60731,7 +58296,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", "supported_modalities": [ @@ -60765,7 +58329,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", "supported_modalities": [ @@ -60797,7 +58360,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", "supported_modalities": [ @@ -60829,7 +58391,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html", "supported_modalities": [ @@ -60861,7 +58422,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html", "supported_modalities": [ @@ -60896,7 +58456,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", "supported_modalities": [ @@ -60937,7 +58496,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, "output_cost_per_token_ultrafast": 0.00033, @@ -60978,7 +58536,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61017,7 +58574,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61064,7 +58620,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, "output_cost_per_token_ultrafast": 0.00033, @@ -61105,7 +58660,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61144,7 +58698,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61189,7 +58742,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_ultrafast": 0.00045, "output_cost_per_token_ultrafast": 0.0003, @@ -61229,7 +58781,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61264,7 +58815,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61302,7 +58852,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61337,7 +58886,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61377,7 +58925,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61417,7 +58964,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61452,7 +58998,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 7e-07, @@ -61478,7 +59023,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 7e-07, @@ -61504,7 +59048,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_batches": 4e-08, "output_cost_per_token_priority": 1.4e-07, @@ -61534,7 +59077,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1048576, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06, @@ -61562,7 +59104,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token_flex": 3.3e-06, "output_cost_per_token_priority": 1.155e-05, @@ -61587,7 +59128,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -61618,7 +59158,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -61656,7 +59195,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -61704,7 +59242,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token_flex": 3.3e-06, "output_cost_per_token_priority": 1.155e-05, @@ -61732,7 +59269,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token_flex": 3e-06, "output_cost_per_token_priority": 1.05e-05, @@ -61747,7 +59283,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -61787,7 +59322,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4.3125e-06, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61803,7 +59337,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 1024000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4.3125e-06, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61821,7 +59354,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 1024000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3.88126e-07, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61838,7 +59370,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.15625e-06, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61852,7 +59383,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -61890,7 +59420,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -61928,7 +59457,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -62023,7 +59551,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -62041,7 +59568,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -62062,7 +59588,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -62092,7 +59617,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -62114,7 +59638,6 @@ } }, "snowflake/claude-sonnet-4-5": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62132,7 +59655,6 @@ "snowflake/claude-sonnet-4-6": { "supports_adaptive_thinking": true, "supports_legacy_thinking": true, - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62148,7 +59670,6 @@ "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-sonnet": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62164,7 +59685,6 @@ "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-opus": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 5e-06, @@ -62181,7 +59701,6 @@ "prompt_cache_min_tokens": 1024 }, "snowflake/claude-haiku-4-5": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 1e-06, @@ -62197,7 +59716,6 @@ "prompt_cache_min_tokens": 4096 }, "snowflake/claude-3-7-sonnet": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62213,7 +59731,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-4.1": { - "max_tokens": 16384, "max_input_tokens": 300000, "max_output_tokens": 16384, "input_cost_per_token": 2e-06, @@ -62228,7 +59745,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-5": { - "max_tokens": 16384, "max_input_tokens": 300000, "max_output_tokens": 16384, "input_cost_per_token": 1.25e-06, @@ -62244,7 +59760,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-5-mini": { - "max_tokens": 16384, "max_input_tokens": 1000000, "max_output_tokens": 16384, "input_cost_per_token": 3e-07, @@ -62256,7 +59771,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-5-nano": { - "max_tokens": 16384, "max_input_tokens": 5000000, "max_output_tokens": 16384, "input_cost_per_token": 1.5e-07, @@ -62268,7 +59782,6 @@ "supports_response_schema": true }, "snowflake/llama4-maverick": { - "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.4e-07, @@ -62279,7 +59792,6 @@ "supports_system_messages": true }, "snowflake/snowflake-arctic-embed-l-v2.0": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 7e-08, "output_cost_per_token": 0.0, @@ -62287,7 +59799,6 @@ "mode": "embedding" }, "snowflake/snowflake-arctic-embed-m-v2.0": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 7e-08, "output_cost_per_token": 0.0, @@ -62297,7 +59808,7 @@ "soniox/stt-async-v4": { "litellm_provider": "soniox", "max_output_tokens": 8000, - "max_tokens": 8000, + "max_input_tokens": 8000, "input_cost_per_second": 0.0, "output_cost_per_second": 2.77778e-05, "mode": "audio_transcription", @@ -62310,7 +59821,7 @@ "soniox/stt-async-v5": { "litellm_provider": "soniox", "max_output_tokens": 8000, - "max_tokens": 8000, + "max_input_tokens": 8000, "input_cost_per_second": 0.0, "output_cost_per_second": 2.77778e-05, "mode": "audio_transcription", @@ -62487,7 +59998,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62541,7 +60051,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62595,7 +60104,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62649,7 +60157,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 2.2e-08, @@ -62703,7 +60210,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62757,7 +60263,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62811,7 +60316,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62865,7 +60369,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 2.2e-08, @@ -62919,7 +60422,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -62945,7 +60447,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -62971,7 +60472,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 2.2e-08, @@ -63026,7 +60526,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -63081,7 +60580,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63106,7 +60604,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63131,7 +60628,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63155,7 +60651,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63179,7 +60674,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63203,7 +60697,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63227,7 +60720,6 @@ "litellm_provider": "tencent", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63251,7 +60743,6 @@ "litellm_provider": "tencent", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63318,7 +60809,6 @@ "source": "https://docs.devin.ai/desktop/models" }, "pinstripes/ps/glm-4.5-air": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.25e-07, @@ -63331,7 +60821,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3.6-35b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.4e-07, @@ -63344,7 +60833,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-30b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-08, @@ -63357,7 +60845,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-coder-30b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-07, @@ -63370,7 +60857,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/deepseek-v4-flash": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1e-07, @@ -63383,7 +60869,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/minimax-m2.7": { - "max_tokens": 1000192, "max_input_tokens": 1000192, "max_output_tokens": 1000192, "input_cost_per_token": 2.55e-07, @@ -63400,7 +60885,6 @@ "litellm_provider": "darkbloom", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.65e-07, "source": "https://www.darkbloom.dev/", @@ -63417,7 +60901,6 @@ "litellm_provider": "darkbloom", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7e-08, "source": "https://www.darkbloom.dev/", @@ -63437,7 +60920,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -63464,7 +60946,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -63493,7 +60974,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.x.ai/v1/language-models", @@ -63566,7 +61046,6 @@ "litellm_provider": "openai", "max_input_tokens": 16000, "max_output_tokens": 2000, - "max_tokens": 2000, "mode": "realtime", "source": "https://developers.openai.com/api/docs/pricing", "supported_modalities": [ @@ -63591,7 +61070,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -63635,7 +61113,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -63678,7 +61155,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "prompt_cache_min_tokens": 512, @@ -63714,7 +61190,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 5e-06, "search_context_cost_per_query": { @@ -63749,7 +61224,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "reasoning_effort_levels": [ @@ -63770,7 +61244,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", @@ -63860,7 +61333,6 @@ "mode": "chat", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, @@ -63959,7 +61431,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 2.1e-05, "output_cost_per_token": 2.1e-05, @@ -63988,7 +61459,6 @@ "litellm_provider": "gemini", "max_input_tokens": 98304, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "audio_transcription", "output_cost_per_token": 1.2e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -64013,7 +61483,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "audio_transcription", "output_cost_per_token": 2.1e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -64094,7 +61563,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64104,7 +61572,6 @@ "input_cost_per_token": 5e-08, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2560, @@ -64114,7 +61581,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64124,7 +61590,6 @@ "input_cost_per_token": 6e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64134,7 +61599,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64144,7 +61608,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64154,7 +61617,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 120000, - "max_tokens": 120000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64164,7 +61626,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64180,7 +61641,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64199,7 +61659,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, @@ -64219,7 +61678,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64239,7 +61697,7 @@ "input_cost_per_token_priority": 2.75e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64257,7 +61715,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_priority": 1.875e-05, @@ -64282,7 +61739,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64301,7 +61757,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, @@ -64321,7 +61776,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64341,7 +61795,7 @@ "input_cost_per_token_priority": 2.75e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64358,7 +61812,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -64375,7 +61828,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -64393,7 +61845,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_priority": 1.875e-05, @@ -64415,7 +61866,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "reasoning_effort_levels": [ @@ -64438,7 +61888,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "output_cost_per_token_priority": 2.8125e-05, @@ -64480,7 +61929,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64500,7 +61948,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_priority": 2.5e-07, @@ -64520,7 +61967,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_priority": 3e-06, @@ -64540,7 +61986,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64575,7 +62020,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_priority": 2.5e-07, @@ -64595,7 +62039,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_priority": 3e-06, @@ -64630,7 +62073,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -64647,7 +62089,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -64663,7 +62104,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "reasoning_effort_levels": [ @@ -64686,7 +62126,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "output_cost_per_token_priority": 2.8125e-05, @@ -64708,7 +62147,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -64725,7 +62163,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 3.96e-06, "source": "https://novita.ai/pricing", @@ -64742,7 +62179,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://novita.ai/pricing", @@ -64759,7 +62195,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 5.8e-07, "source": "https://novita.ai/pricing", @@ -64776,7 +62211,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://novita.ai/pricing", @@ -64793,7 +62227,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://novita.ai/pricing", @@ -64810,7 +62243,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -64827,7 +62259,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://novita.ai/pricing", @@ -64844,7 +62275,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://novita.ai/pricing", @@ -64860,7 +62290,6 @@ "litellm_provider": "novita", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://novita.ai/pricing", @@ -64877,7 +62306,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://novita.ai/pricing", @@ -64894,7 +62322,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 3.2000000000000003e-06, "source": "https://novita.ai/pricing", @@ -64911,7 +62338,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://novita.ai/pricing", @@ -64927,7 +62353,6 @@ "litellm_provider": "novita", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://novita.ai/pricing", @@ -64944,7 +62369,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://novita.ai/pricing", @@ -64960,7 +62384,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.6e-06, "source": "https://novita.ai/pricing", @@ -64976,7 +62399,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://novita.ai/pricing", @@ -64992,7 +62414,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.0000000000000002e-07, "source": "https://novita.ai/pricing", @@ -65008,7 +62429,6 @@ "litellm_provider": "novita", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.13e-06, "source": "https://novita.ai/pricing", @@ -65024,7 +62444,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.3600000000000004e-07, "source": "https://novita.ai/pricing", @@ -65041,7 +62460,6 @@ "litellm_provider": "novita", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://novita.ai/pricing", @@ -65058,7 +62476,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.044e-06, "source": "https://novita.ai/pricing", @@ -65074,7 +62491,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.6000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65090,7 +62506,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.4e-06, "source": "https://novita.ai/pricing", @@ -65107,7 +62522,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://novita.ai/pricing", @@ -65124,7 +62538,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65141,7 +62554,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65157,7 +62569,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.0000000000000003e-07, "source": "https://novita.ai/pricing", @@ -65172,7 +62583,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.0000000000000003e-07, "source": "https://novita.ai/pricing", @@ -65188,7 +62598,6 @@ "litellm_provider": "novita", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65205,7 +62614,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://novita.ai/pricing", @@ -65222,7 +62630,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://novita.ai/pricing", @@ -65238,7 +62645,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://novita.ai/pricing", @@ -65253,7 +62659,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.2000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65268,7 +62673,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://novita.ai/pricing", @@ -65283,7 +62687,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.6000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65299,7 +62702,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://novita.ai/pricing", @@ -65316,7 +62718,6 @@ "litellm_provider": "novita", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.2000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65332,7 +62733,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://novita.ai/pricing", @@ -65346,7 +62746,6 @@ "litellm_provider": "novita", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-08, "source": "https://novita.ai/pricing", @@ -65358,7 +62757,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://novita.ai/pricing", @@ -65375,7 +62773,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65392,7 +62789,6 @@ "litellm_provider": "novita", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.0000000000000003e-07, "source": "https://novita.ai/pricing", @@ -65408,7 +62804,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.4850000000000002e-06, "source": "https://novita.ai/pricing", @@ -65423,7 +62818,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 8.900000000000001e-07, "source": "https://api.novita.ai/v3/openai/models", @@ -65436,7 +62830,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65450,7 +62843,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8.900000000000001e-07, "source": "https://api.novita.ai/v3/openai/models", @@ -65463,7 +62855,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65477,7 +62868,6 @@ "litellm_provider": "novita", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.66e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65491,7 +62881,6 @@ "litellm_provider": "novita", "max_input_tokens": 131000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2e-08, "source": "https://api.novita.ai/v3/openai/models", @@ -65500,7 +62889,7 @@ }, "wandb/deepseek-ai/DeepSeek-V4-Flash": { "supports_reasoning": true, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1049000, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, @@ -65514,7 +62903,7 @@ }, "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 1.3e-07, "output_cost_per_token": 2.8e-07, @@ -65527,7 +62916,7 @@ }, "wandb/deepseek-ai/DeepSeek-V4-Pro": { "supports_reasoning": true, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1049000, "input_cost_per_token": 1.15e-06, "output_cost_per_token": 2.55e-06, @@ -65541,7 +62930,7 @@ }, "wandb/google/gemma-4-31B-it": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3.4e-07, @@ -65552,7 +62941,7 @@ }, "wandb/ibm-granite/granite-4.1-8b": { "deprecation_date": "2026-10-05", - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, @@ -65563,7 +62952,7 @@ }, "wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": { "deprecation_date": "2026-10-05", - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, @@ -65574,7 +62963,7 @@ }, "wandb/meta-llama/Llama-3.1-70B-Instruct": { "deprecation_date": "2026-10-05", - "max_tokens": 128000, + "max_output_tokens": 128000, "max_input_tokens": 131000, "input_cost_per_token": 8e-07, "output_cost_per_token": 8e-07, @@ -65585,7 +62974,7 @@ }, "wandb/MiniMaxAI/MiniMax-M3": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 2.3e-07, "output_cost_per_token": 9.6e-07, @@ -65598,7 +62987,7 @@ }, "wandb/moonshotai/Kimi-K2.7-Code": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 7.1e-07, "output_cost_per_token": 3.5e-06, @@ -65611,7 +63000,7 @@ }, "wandb/moonshotai/Kimi-K2.6": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 6.5e-07, "output_cost_per_token": 3.41e-06, @@ -65624,7 +63013,7 @@ }, "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 7e-08, "output_cost_per_token": 2e-07, @@ -65637,7 +63026,7 @@ }, "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.15e-06, @@ -65650,7 +63039,7 @@ }, "wandb/OpenPipe/Qwen3-14B-Instruct": { "deprecation_date": "2026-10-05", - "max_tokens": 32768, + "max_output_tokens": 32768, "max_input_tokens": 32800, "input_cost_per_token": 5e-08, "output_cost_per_token": 2.2e-07, @@ -65661,7 +63050,7 @@ }, "wandb/Qwen/Qwen3.8-27B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 4e-07, "output_cost_per_token": 3e-06, @@ -65674,7 +63063,7 @@ }, "wandb/Qwen/Qwen3.6-35B-A3B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.25e-06, @@ -65685,7 +63074,7 @@ }, "wandb/Qwen/Qwen3.6-27B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 6e-07, "output_cost_per_token": 3.6e-06, @@ -65700,7 +63089,7 @@ "wandb/Qwen/Qwen3.5-35B-A3B": { "deprecation_date": "2026-10-05", "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.25e-06, @@ -65711,7 +63100,7 @@ }, "wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": { "deprecation_date": "2026-10-05", - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3e-07, @@ -65744,7 +63133,7 @@ }, "wandb/zai-org/GLM-5.2": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 1049000, "input_cost_per_token": 7.6e-07, "output_cost_per_token": 2.42e-06, @@ -65756,7 +63145,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "deepinfra/openai/gpt-oss-120b-Turbo": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, @@ -65770,7 +63159,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/MiniMaxAI/MiniMax-M2.7": { - "max_tokens": 196608, + "max_output_tokens": 196608, "max_input_tokens": 196608, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1e-06, @@ -65786,7 +63175,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.8-27B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4e-07, "output_cost_per_token": 3e-06, @@ -65802,7 +63191,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-31B-it-Ultra": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2.7e-07, "output_cost_per_token": 7.6e-07, @@ -65816,7 +63205,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K2.5": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4.5e-07, "output_cost_per_token": 2.25e-06, @@ -65832,7 +63221,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.7-Flash": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 6e-08, "output_cost_per_token": 4e-07, @@ -65848,7 +63237,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.6": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 5e-07, "output_cost_per_token": 2e-06, @@ -65867,7 +63256,7 @@ "input_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-05, "prompt_cache_min_tokens": 1024, @@ -65881,7 +63270,7 @@ "supports_vision": true }, "deepinfra/anthropic/claude-sonnet-4-6": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -65897,7 +63286,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemini-3.5-flash": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 1.5e-06, "output_cost_per_token": 9e-06, @@ -65912,7 +63301,7 @@ "supports_audio_input": true }, "deepinfra/XiaomiMiMo/MiMo-V2.5": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4e-07, "output_cost_per_token": 2e-06, @@ -65928,7 +63317,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Max": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1.2e-06, "output_cost_per_token": 6e-06, @@ -65943,7 +63332,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-31B-it-turbo": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 9e-08, "output_cost_per_token": 3.4e-07, @@ -65958,7 +63347,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/thinkingmachines/Inkling-Small": { - "max_tokens": 524288, + "max_output_tokens": 524288, "max_input_tokens": 524288, "input_cost_per_token": 4.5e-07, "output_cost_per_token": 1.2e-06, @@ -65974,7 +63363,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-models/Muse-Glimmer-30B": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, @@ -65990,7 +63379,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Max-Thinking": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1.2e-06, "output_cost_per_token": 6e-06, @@ -66005,7 +63394,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-VL-235B-A22B-Instruct": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2e-07, "output_cost_per_token": 8.8e-07, @@ -66020,7 +63409,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-VL-30B-A3B-Instruct": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, @@ -66033,7 +63422,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-27B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.6e-07, "output_cost_per_token": 2.6e-06, @@ -66047,7 +63436,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.6-35B-A3B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, "output_cost_per_token": 9.5e-07, @@ -66061,7 +63450,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/nvidia/Nemotron-Content-Safety-3.5": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2e-07, "output_cost_per_token": 2e-07, @@ -66074,7 +63463,7 @@ "input_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-05, "prompt_cache_min_tokens": 512, @@ -66088,7 +63477,7 @@ "supports_vision": true }, "deepinfra/thinkingmachines/Inkling": { - "max_tokens": 524288, + "max_output_tokens": 524288, "max_input_tokens": 524288, "input_cost_per_token": 9.5e-07, "output_cost_per_token": 4.05e-06, @@ -66104,7 +63493,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K2.6": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 3.5e-06, @@ -66120,7 +63509,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Pro-0813": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.3e-06, "output_cost_per_token": 2.6e-06, @@ -66135,7 +63524,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.7-Max": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 2.5e-06, "output_cost_per_token": 7.5e-06, @@ -66150,7 +63539,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-2.0-mini": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1e-07, "output_cost_per_token": 4e-07, @@ -66166,7 +63555,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.8-2.4T-A95B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, @@ -66182,7 +63571,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/MiniMaxAI/MiniMax-M3": { - "max_tokens": 524288, + "max_output_tokens": 524288, "max_input_tokens": 524288, "input_cost_per_token": 2.8e-07, "output_cost_per_token": 1.1e-06, @@ -66198,7 +63587,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemini-3.1-flash-lite": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.5e-06, @@ -66213,7 +63602,7 @@ "supports_audio_input": true }, "deepinfra/google/gemini-3.7-flash": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 3.75e-06, @@ -66229,7 +63618,7 @@ "supports_audio_input": true }, "deepinfra/inclusionAI/Ling-3.0-flash": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 6e-08, "output_cost_per_token": 1.8e-07, @@ -66245,7 +63634,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/stepfun-ai/Step-3.7-Flash": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2e-07, "output_cost_per_token": 1.15e-06, @@ -66259,7 +63648,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-35B-A3B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 1e-06, @@ -66275,7 +63664,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-1.8": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06, @@ -66291,7 +63680,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/tencent/Hy3": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 5.8e-07, @@ -66307,7 +63696,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-2.0-code": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 5e-07, "output_cost_per_token": 3e-06, @@ -66323,7 +63712,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-2.0-pro": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 5e-07, "output_cost_per_token": 3e-06, @@ -66339,7 +63728,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-5": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 6e-07, "output_cost_per_token": 2.08e-06, @@ -66355,7 +63744,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/nvidia/Nemotron-3-Nano-30B-A3B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 5e-08, "output_cost_per_token": 2e-07, @@ -66370,7 +63759,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K2.7-Code": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 6.8e-07, "output_cost_per_token": 3.4e-06, @@ -66389,7 +63778,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1e-05, "prompt_cache_min_tokens": 1024, @@ -66403,7 +63792,7 @@ "supports_vision": true }, "deepinfra/Qwen/Qwen3.5-397B-A17B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4.5e-07, "output_cost_per_token": 3e-06, @@ -66419,7 +63808,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 8e-08, "output_cost_per_token": 1.8e-07, @@ -66435,7 +63824,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-E4B-it": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2e-08, "output_cost_per_token": 1e-07, @@ -66448,7 +63837,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.2": { - "max_tokens": 163840, + "max_output_tokens": 163840, "max_input_tokens": 163840, "input_cost_per_token": 2.6e-07, "output_cost_per_token": 3.8e-07, @@ -66463,7 +63852,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.8-Max": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1.65e-06, "output_cost_per_token": 4.951e-06, @@ -66481,7 +63870,7 @@ "input_cost_per_token": 1e-05, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 5e-05, "prompt_cache_min_tokens": 512, @@ -66496,7 +63885,7 @@ "thinking_always_on": true }, "deepinfra/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.2e-06, @@ -66512,7 +63901,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-122B-A10B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.9e-07, "output_cost_per_token": 2.4e-06, @@ -66526,7 +63915,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-5.1": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 1.05e-06, "output_cost_per_token": 3.5e-06, @@ -66542,7 +63931,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Pro": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.3e-06, "output_cost_per_token": 2.6e-06, @@ -66557,7 +63946,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 8.5e-08, "output_cost_per_token": 4e-07, @@ -66571,7 +63960,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-5.2": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 2.4e-06, @@ -66587,7 +63976,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K3": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 2.85e-06, "output_cost_per_token": 1.425e-05, @@ -66605,7 +63994,7 @@ "input_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-05, "prompt_cache_min_tokens": 2048, @@ -66619,7 +64008,7 @@ "supports_vision": true }, "deepinfra/Qwen/Qwen3.6-27B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 3.2e-07, "output_cost_per_token": 3.2e-06, @@ -66633,7 +64022,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-26B-A4B-it": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7e-08, "output_cost_per_token": 3.4e-07, @@ -66647,7 +64036,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemini-3.1-pro": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 2e-06, "output_cost_per_token": 1.2e-05, @@ -66662,7 +64051,7 @@ "supports_audio_input": true }, "deepinfra/XiaomiMiMo/MiMo-V2.5-Pro": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1e-06, "output_cost_per_token": 3e-06, @@ -66678,7 +64067,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-haiku-4-5": { - "max_tokens": 200000, + "max_output_tokens": 200000, "max_input_tokens": 200000, "input_cost_per_token": 1e-06, "output_cost_per_token": 5e-06, @@ -66693,7 +64082,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Flash": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 9e-08, "output_cost_per_token": 1.8e-07, @@ -66709,7 +64098,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4.1-Flash": { - "max_tokens": 131072, "max_input_tokens": 1048576, "max_output_tokens": 131072, "input_cost_per_token": 1.4e-07, @@ -66726,7 +64114,6 @@ "source": "https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash" }, "deepinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017": { - "max_tokens": 131072, "max_input_tokens": 1048576, "max_output_tokens": 131072, "input_cost_per_token": 1.4e-07, @@ -66743,7 +64130,7 @@ "source": "https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash-1017" }, "deepinfra/openai/gpt-oss-120b-Ultra": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2e-07, "output_cost_per_token": 9.5e-07, @@ -66757,7 +64144,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-9B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, "output_cost_per_token": 1.5e-07, @@ -66770,7 +64157,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/MiniMaxAI/MiniMax-M2.7-Turbo": { - "max_tokens": 196608, + "max_output_tokens": 196608, "max_input_tokens": 196608, "input_cost_per_token": 3.8e-07, "output_cost_per_token": 1.7e-06, @@ -66786,7 +64173,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.7": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 4e-07, "output_cost_per_token": 1.75e-06, @@ -66802,7 +64189,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-31B-it": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.3e-07, "output_cost_per_token": 3.8e-07, @@ -66821,7 +64208,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -66856,7 +64242,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -66884,7 +64269,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -66912,7 +64296,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67188,7 +64571,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67215,7 +64597,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67242,7 +64623,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67273,7 +64653,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67301,7 +64680,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://console.groq.com/docs/model/qwen/qwen3.8-27b", @@ -67317,7 +64695,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -67338,7 +64715,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -67359,7 +64735,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -67380,7 +64755,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "reasoning_effort_levels": [ @@ -67401,7 +64775,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -67416,7 +64789,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -67431,7 +64803,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://mistral.ai/news/devstral-2-vibe-cli", @@ -67532,7 +64903,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", @@ -67548,7 +64918,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -67567,7 +64936,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.6e-06, "output_cost_per_token_priority": 8.25e-06, @@ -67586,7 +64954,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -67605,7 +64972,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.6e-06, "output_cost_per_token_priority": 8.25e-06, @@ -67621,7 +64987,7 @@ "input_cost_per_token": 2.1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -67663,7 +65029,7 @@ "input_cost_per_token": 2.1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -67680,7 +65046,7 @@ "input_cost_per_token_priority": 1.875e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_priority": 6.25e-07, @@ -67698,7 +65064,7 @@ "input_cost_per_token_priority": 2.8125e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 7.5e-07, "output_cost_per_token_priority": 9.375e-07, @@ -67715,7 +65081,7 @@ "input_cost_per_token_priority": 1.875e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_priority": 6.25e-07, @@ -67733,7 +65099,7 @@ "input_cost_per_token_priority": 2.8125e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 7.5e-07, "output_cost_per_token_priority": 9.375e-07, @@ -67751,7 +65117,7 @@ "input_cost_per_token_priority": 1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.05e-06, "output_cost_per_token_priority": 4.05e-06, @@ -67767,7 +65133,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 40960, "max_output_tokens": 40960, - "max_tokens": 40960, "mode": "embedding", "source": "https://api.fireworks.ai/v1/serverless/models" }, @@ -67790,7 +65155,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://api.together.ai/v1/models" @@ -67850,7 +65215,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 131072, "max_output_tokens": 40960, - "max_tokens": 40960, "mode": "chat", "output_cost_per_token": 1.49e-06, "source": "https://api.cerebras.ai/public/v1/models/gemma-4-31b", @@ -67876,7 +65240,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", @@ -67890,7 +65253,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", @@ -67906,7 +65268,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -67929,7 +65290,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true, @@ -67971,7 +65331,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -67986,7 +65345,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68001,7 +65359,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68016,7 +65373,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.25e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68032,7 +65388,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68049,7 +65404,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68062,7 +65416,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68137,7 +65490,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68155,7 +65507,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", @@ -68171,7 +65522,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", @@ -68186,7 +65536,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68204,7 +65553,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", @@ -68224,7 +65572,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", @@ -68249,7 +65596,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, @@ -68288,7 +65634,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, @@ -68326,7 +65671,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, @@ -68363,7 +65707,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -68401,7 +65744,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_adaptive_thinking": true, @@ -68436,7 +65778,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68454,7 +65795,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", @@ -68470,7 +65810,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", @@ -68485,7 +65824,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68503,7 +65841,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", @@ -68523,7 +65860,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", @@ -68548,7 +65884,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, @@ -68582,7 +65917,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, @@ -68615,7 +65949,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, @@ -68647,7 +65980,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -68680,7 +66012,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_adaptive_thinking": true, @@ -68709,7 +66040,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68745,7 +66075,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68781,7 +66110,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68812,7 +66140,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1048576, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -68836,7 +66163,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -68855,7 +66181,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68874,7 +66199,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68893,7 +66217,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68913,7 +66236,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ @@ -68933,7 +66255,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ @@ -68956,7 +66277,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -68992,7 +66312,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -69021,7 +66340,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -69054,7 +66372,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -69074,7 +66391,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ @@ -69094,7 +66410,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ @@ -69118,7 +66433,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 8.4e-07, "output_cost_per_token_priority": 2.94e-06, @@ -69140,7 +66454,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_priority": 4.38e-06, @@ -69162,7 +66475,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 4.4e-07, "output_cost_per_token_priority": 1.54e-06, @@ -69184,7 +66496,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 32000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -69207,7 +66518,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -69229,7 +66539,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 9e-07, "output_cost_per_token_priority": 3.15e-06, @@ -69250,7 +66559,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 6e-07, "output_cost_per_token_priority": 2.1e-06, @@ -69272,7 +66580,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 1.33e-06, "output_cost_per_token_priority": 4.66e-06, @@ -69291,7 +66598,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.375e-05, "supported_endpoints": [ @@ -69325,7 +66631,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.05e-06, "supports_prompt_caching": true, @@ -69338,7 +66643,6 @@ "input_cost_per_token": 1.63e-07, "litellm_provider": "azure", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -69346,7 +66650,6 @@ "input_cost_per_token": 2.5e-08, "litellm_provider": "azure", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -69377,7 +66680,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_image": 0.04, "output_cost_per_token": 0, @@ -69402,7 +66704,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -69427,7 +66728,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -69452,7 +66752,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/models/lyria-realtime-exp", @@ -69825,7 +67124,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -69849,7 +67147,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://inference.baseten.co/v1/models", @@ -69874,7 +67171,7 @@ "output_cost_per_token": 1.5e-07, "source": "https://api.together.ai/v1/models", "max_input_tokens": 128000, - "max_tokens": 128000 + "max_output_tokens": 128000 }, "vertex_ai/gemini-2.5-flash-native-audio": { "deprecation_date": "2026-12-13", @@ -69935,7 +67232,7 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai", "max_output_tokens": 57920, - "max_tokens": 57920, + "max_input_tokens": 57920, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -69998,7 +67295,7 @@ "output_cost_per_token": 3.5e-06, "source": "https://api.together.ai/v1/models", "max_input_tokens": 4096, - "max_tokens": 4096 + "max_output_tokens": 4096 }, "together_ai/meta-llama/Llama-3.2-1B-Instruct": { "input_cost_per_token": 6e-08, @@ -70007,7 +67304,7 @@ "output_cost_per_token": 6e-08, "source": "https://api.together.ai/v1/models", "max_input_tokens": 131072, - "max_tokens": 131072 + "max_output_tokens": 131072 }, "together_ai/meta-llama/Llama-3.2-3B-Instruct": { "input_cost_per_token": 6e-08, @@ -70016,7 +67313,7 @@ "output_cost_per_token": 6e-08, "source": "https://api.together.ai/v1/models", "max_input_tokens": 131072, - "max_tokens": 131072 + "max_output_tokens": 131072 }, "together_ai/Qwen/Qwen2-1.5B-Instruct": { "input_cost_per_token": 2e-08, @@ -70025,7 +67322,7 @@ "output_cost_per_token": 2e-08, "source": "https://api.together.ai/v1/models", "max_input_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "together_ai/Qwen/Qwen2.5-14B-Instruct": { "input_cost_per_token": 8e-07, @@ -70034,7 +67331,7 @@ "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models", "max_input_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "together_ai/Qwen/Qwen2.5-72B-Instruct": { "input_cost_per_token": 1.2e-06, @@ -70043,14 +67340,13 @@ "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", "max_input_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "together_ai/Salesforce/Llama-Rank-V1": { "deprecation_date": "2026-02-06", "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://api.together.ai/v1/models" @@ -70059,7 +67355,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 16384, "mode": "completion", "output_cost_per_token": 2e-07, "source": "https://api.together.xyz/v1/models" @@ -70077,7 +67373,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://api.together.ai/v1/models" @@ -70087,7 +67383,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70097,7 +67393,7 @@ "input_cost_per_token": 9e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://api.together.ai/v1/models" @@ -70107,7 +67403,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70117,7 +67413,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70127,7 +67423,7 @@ "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "max_input_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models" @@ -70137,7 +67433,7 @@ "input_cost_per_token": 1.95e-06, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://api.together.ai/v1/models" @@ -70147,7 +67443,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.together.ai/v1/models" @@ -70157,7 +67453,7 @@ "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70167,7 +67463,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models" @@ -70177,7 +67473,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models" @@ -70187,7 +67483,7 @@ "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models" @@ -70197,7 +67493,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://api.together.ai/v1/models" @@ -70208,7 +67504,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.6e-06, "source": "https://api.together.ai/v1/models" @@ -70218,7 +67514,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.together.ai/v1/models" @@ -70228,7 +67524,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://api.together.ai/v1/models" @@ -70238,7 +67534,7 @@ "input_cost_per_token": 1.6e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://api.together.ai/v1/models" @@ -70248,7 +67544,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.7e-06, "source": "https://api.together.ai/v1/models" @@ -70258,7 +67554,7 @@ "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "max_input_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models" @@ -70268,7 +67564,7 @@ "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models" @@ -70278,7 +67574,7 @@ "input_cost_per_token": 3.9e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 9.7e-07, "source": "https://api.together.ai/v1/models" @@ -70288,7 +67584,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70298,7 +67594,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.9e-07, "source": "https://api.together.ai/v1/models" @@ -70308,7 +67604,7 @@ "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://api.together.ai/v1/models" @@ -70318,7 +67614,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70328,7 +67624,7 @@ "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://api.together.ai/v1/models" @@ -70338,7 +67634,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://api.together.ai/v1/models" @@ -70348,7 +67644,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70358,7 +67654,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://api.together.ai/v1/models" @@ -70368,7 +67664,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://api.together.ai/v1/models" @@ -70379,7 +67675,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://api.together.ai/v1/models" @@ -70390,7 +67686,7 @@ "input_cost_per_token": 9.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.together.ai/v1/models" @@ -70400,7 +67696,7 @@ "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://api.together.ai/v1/models" @@ -70411,7 +67707,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 512288, - "max_tokens": 512288, + "max_output_tokens": 512288, "mode": "chat", "output_cost_per_token": 3.6e-06, "source": "https://api.together.ai/v1/models" @@ -70421,7 +67717,7 @@ "input_cost_per_token": 5e-08, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70431,7 +67727,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.1e-06, "source": "https://api.together.ai/v1/models" @@ -70441,7 +67737,7 @@ "input_cost_per_token": 4.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.together.ai/v1/models" @@ -70451,7 +67747,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://api.together.ai/v1/models" @@ -70462,7 +67758,7 @@ "input_cost_per_token": 1.4e-06, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.together.ai/v1/models" @@ -70529,7 +67825,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_batches": 8.25e-06, @@ -70702,7 +67997,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_above_272k_tokens": 9e-05, @@ -70746,7 +68040,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_above_272k_tokens": 9e-07, @@ -70795,7 +68088,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -70851,7 +68143,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -70865,7 +68156,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -70888,7 +68178,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -70902,7 +68191,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -70937,7 +68225,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -70959,7 +68246,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -71035,7 +68321,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_batches": 8.25e-06, @@ -71248,7 +68533,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -71296,7 +68580,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "audio_transcription", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ @@ -71337,7 +68620,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ @@ -71355,7 +68637,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 512000, "max_output_tokens": 65500, - "max_tokens": 65500, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71368,7 +68649,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71381,7 +68661,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 200000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71396,7 +68675,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71417,7 +68695,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71435,7 +68712,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71455,7 +68731,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71476,7 +68751,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71495,7 +68769,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71510,7 +68783,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.61333e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71539,7 +68811,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.6e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71555,7 +68826,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71569,7 +68839,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71584,7 +68853,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.84e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71599,7 +68867,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.38e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71614,7 +68881,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.411e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71630,7 +68896,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.4246e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71647,7 +68912,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.0136e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71664,7 +68928,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.411e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71681,7 +68944,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.32375e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71698,7 +68960,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 119000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.5353e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71711,7 +68972,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71728,7 +68988,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71745,7 +69004,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71762,7 +69020,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71779,7 +69036,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.499999e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71797,7 +69053,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71814,7 +69069,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 3.9998e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71826,7 +69080,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 3.9998e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71839,7 +69092,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 7.889e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71854,7 +69106,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.9438e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71869,7 +69120,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.9438e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71885,7 +69135,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 200000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.09848e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71899,7 +69148,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71915,7 +69163,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71932,7 +69179,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71949,7 +69195,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71966,7 +69211,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71983,7 +69227,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71999,7 +69242,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 0.00018, "source": "https://aihubmix.com/api/v1/models", @@ -72016,7 +69258,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72033,7 +69274,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72050,7 +69290,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72066,7 +69305,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72111,7 +69349,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72127,7 +69364,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72143,7 +69379,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.248e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72159,7 +69394,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.535e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72175,7 +69409,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.9995e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72191,7 +69424,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.999e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72207,7 +69439,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72223,7 +69454,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.0984e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72236,7 +69466,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72272,7 +69501,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 204800, "max_output_tokens": 204800, - "max_tokens": 204800, "mode": "chat", "output_cost_per_token": 1.1832e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72286,7 +69514,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.152e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72311,7 +69538,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 5.48e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72323,7 +69549,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9.008e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72338,7 +69563,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9.864e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72353,7 +69577,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.532e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72368,7 +69591,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.524e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72385,7 +69607,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.608e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72402,7 +69623,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.128e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72419,7 +69639,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72436,7 +69655,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.80025e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72454,7 +69672,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5.07e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72639,7 +69856,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "wandb", "max_input_tokens": 1049000, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.5e-07, "source": "https://wandb.ai/site/pricing/tokens/", @@ -72652,7 +69869,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "wandb", "max_input_tokens": 262000, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://wandb.ai/site/pricing/tokens/", @@ -72693,7 +69910,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, @@ -72725,7 +69941,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_flex": 7.5e-06, @@ -72757,7 +69972,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, @@ -72779,7 +69993,6 @@ "litellm_provider": "xiaomi_mimo", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.7e-07, "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md", @@ -72802,7 +70015,6 @@ "litellm_provider": "xiaomi_mimo", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md", @@ -72827,7 +70039,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -72853,7 +70064,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72875,7 +70085,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72897,7 +70106,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72919,7 +70127,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72940,7 +70147,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72962,7 +70168,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72983,7 +70188,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73005,7 +70209,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73027,7 +70230,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73048,7 +70250,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73070,7 +70271,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73092,7 +70292,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73113,7 +70312,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73135,7 +70333,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73160,7 +70357,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73185,7 +70381,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73210,7 +70405,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73231,7 +70425,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73253,7 +70446,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -73271,7 +70463,6 @@ "litellm_provider": "baseten", "max_input_tokens": 262000, "max_output_tokens": 262000, - "max_tokens": 262000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73289,7 +70480,6 @@ "litellm_provider": "baseten", "max_input_tokens": 262000, "max_output_tokens": 262000, - "max_tokens": 262000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73306,7 +70496,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://inference.baseten.co/v1/models", @@ -73323,7 +70512,6 @@ "litellm_provider": "baseten", "max_input_tokens": 202800, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73341,7 +70529,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4.05e-06, "source": "https://inference.baseten.co/v1/models", @@ -73359,7 +70546,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -73376,7 +70562,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73393,7 +70578,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inference.baseten.co/v1/models", @@ -73410,7 +70594,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.6e-07, "source": "https://inference.baseten.co/v1/models", @@ -73428,7 +70611,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.48e-06, "source": "https://inference.baseten.co/v1/models", @@ -73445,7 +70627,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.96e-06, "source": "https://inference.baseten.co/v1/models", @@ -73462,7 +70643,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://inference.baseten.co/v1/models", @@ -73482,7 +70662,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, @@ -73503,7 +70682,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, @@ -73524,7 +70702,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, @@ -73545,7 +70722,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.2e-06, @@ -73563,7 +70739,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 196608, "max_output_tokens": 196608, - "max_tokens": 196608, "mode": "chat", "output_cost_per_token_priority": 1.2e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -73629,7 +70804,6 @@ "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -73647,7 +70821,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -73666,7 +70839,6 @@ "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo", @@ -73682,7 +70854,6 @@ "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -73699,7 +70870,6 @@ "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.5, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo", @@ -73746,7 +70916,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73779,7 +70948,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73812,7 +70980,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73845,7 +71012,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73883,7 +71049,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73916,7 +71081,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73952,7 +71116,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -73991,7 +71154,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74031,7 +71193,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74069,7 +71230,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74108,7 +71268,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74141,7 +71300,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "supports_mid_conversation_system": true, @@ -74179,7 +71337,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -74199,7 +71356,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -74219,7 +71375,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -74241,7 +71396,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", @@ -74254,7 +71408,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -74267,7 +71420,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", @@ -74303,7 +71455,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -74341,7 +71492,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "supports_mid_conversation_system": true, @@ -74387,7 +71537,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74429,7 +71578,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -74466,7 +71614,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -74503,7 +71650,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -74544,7 +71690,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74591,7 +71736,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -74633,7 +71777,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74765,7 +71908,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -74811,7 +71953,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74856,7 +71997,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -74896,7 +72036,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -74929,7 +72068,6 @@ "litellm_provider": "prism", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 6.3e-07, "source": "https://prisminference.com/pricing", @@ -74954,7 +72092,6 @@ "litellm_provider": "prism", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.1e-07, "source": "https://prisminference.com/pricing", @@ -74987,7 +72124,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_flex": 3e-06, @@ -75013,7 +72149,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6.6e-06, "output_cost_per_token_flex": 3.3e-06, @@ -75031,7 +72166,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -75046,10 +72180,14 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": 6e-05, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-06, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, "cache_creation_input_token_cost_above_12hr_priority": 1e-05, + "cache_creation_input_token_cost_above_12hr_ultrafast": 3e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, @@ -75082,7 +72220,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -75150,7 +72287,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -75196,7 +72332,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "responses", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -75244,7 +72379,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -75279,7 +72413,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 9e-06, "output_cost_per_token": 9e-06, @@ -75293,7 +72426,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 6e-06, "output_cost_per_token": 6e-06, @@ -75310,7 +72442,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -75328,7 +72459,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", @@ -75361,7 +72491,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.28e-06, "output_cost_per_token_batches": 2.64e-06, @@ -75394,7 +72523,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.808e-06, "output_cost_per_token_batches": 2.904e-06, @@ -75417,7 +72545,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2.75e-06, @@ -75432,7 +72559,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/model-router/", @@ -75452,7 +72578,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -75497,7 +72622,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -75543,7 +72667,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -75587,7 +72710,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -75615,7 +72737,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, @@ -75648,7 +72769,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -75702,7 +72822,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -75749,7 +72868,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -75794,7 +72912,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "search_context_cost_per_query": { @@ -75837,7 +72954,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "supports_mid_conversation_system": true, @@ -75888,7 +73004,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -75935,7 +73050,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "search_context_cost_per_query": { @@ -75977,7 +73091,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -76019,7 +73132,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -76065,7 +73177,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -76117,7 +73228,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "search_context_cost_per_query": { @@ -76164,7 +73274,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -76223,7 +73332,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -76274,7 +73382,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -76324,7 +73431,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -76369,7 +73475,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -76408,7 +73513,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 261120, "max_output_tokens": 98304, - "max_tokens": 98304, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", @@ -76419,7 +73523,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 261120, "max_output_tokens": 98304, - "max_tokens": 98304, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", @@ -76430,7 +73533,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 261120, "max_output_tokens": 98304, - "max_tokens": 98304, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", @@ -76439,9 +73541,10 @@ "azure_ai/Microsoft-Decision-1": { "input_cost_per_token": 4.2e-08, "litellm_provider": "azure_ai", + "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" }, "gpt-rosalind-discovery": { "cache_read_input_token_cost": 5e-07, @@ -76465,7 +73568,6 @@ "input_cost_per_token": 8e-08, "litellm_provider": "azure_ai", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048.0, @@ -76485,7 +73587,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048.0, @@ -76554,7 +73655,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, @@ -76600,7 +73700,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, @@ -76650,7 +73749,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_reasoning_token": 1e-05, "output_cost_per_token": 1e-05, @@ -76709,7 +73807,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_reasoning_token": 1e-05, "output_cost_per_token": 1e-05, @@ -76767,7 +73864,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_reasoning_token": 2e-05, "output_cost_per_token": 2e-05, @@ -76815,7 +73911,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -76828,7 +73923,6 @@ "supports_vision": true }, "openinfer/@oi/GLM-5.3-Flash": { - "max_tokens": 393216, "max_input_tokens": 393216, "max_output_tokens": 393216, "input_cost_per_token": 1.95e-07, @@ -76840,7 +73934,6 @@ "source": "https://api.openinfer.ai/v1/models" }, "openinfer/@oi/Qwen3.5-27B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.95e-07, @@ -76852,7 +73945,6 @@ "source": "https://api.openinfer.ai/v1/models" }, "openinfer/@oi/Qwen3.5-9B": { - "max_tokens": 100000, "max_input_tokens": 100000, "max_output_tokens": 100000, "input_cost_per_token": 5e-06, @@ -76931,7 +74023,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://api.together.ai/v1/models", @@ -76954,7 +74046,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.311e-06, "output_cost_per_token_batches": 1.6555e-06, @@ -76976,7 +74067,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -77000,7 +74090,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-05, "prompt_cache_min_tokens": 512.0, @@ -77030,7 +74119,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.25e-06, "source": "https://inference.baseten.co/v1/models", @@ -77048,7 +74136,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inference.baseten.co/v1/models", @@ -77064,7 +74151,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "wandb", "max_input_tokens": 1049000, - "max_tokens": 943718, + "max_output_tokens": 943718, "mode": "chat", "output_cost_per_token": 7.5e-07, "source": "https://openrouter.ai/api/v1/models/nvidia/nemotron-3.5-super-vl/endpoints", @@ -77080,7 +74167,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -77094,7 +74181,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -77108,7 +74194,7 @@ "input_cost_per_token": 2.5e-07, "litellm_provider": "nebius", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://tokenfactory.nebius.com/models/catalog/text2text/reflection%2Fbeam", @@ -77121,7 +74207,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -77138,7 +74223,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -77157,7 +74241,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, @@ -77177,7 +74260,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -77199,7 +74281,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.25e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -77221,7 +74302,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -77243,7 +74323,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -77265,7 +74344,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -77284,7 +74362,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -77321,7 +74399,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 9e-07, @@ -77357,7 +74434,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_priority": 2.25e-06, @@ -77406,7 +74482,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -77436,7 +74511,6 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07, - "max_tokens": 500000, "max_output_tokens": 500000, "max_input_tokens": 500000, "supports_function_calling": true, @@ -77474,7 +74548,6 @@ "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, "cache_creation_input_token_cost": 1.25e-07, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 922000, "supports_function_calling": true, @@ -77514,7 +74587,6 @@ "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 2e-08, "cache_creation_input_token_cost": 2.5e-07, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 922000, "supports_function_calling": true, @@ -77554,7 +74626,6 @@ "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, "cache_creation_input_token_cost": 1.25e-07, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77593,7 +74664,6 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05, "cache_read_input_token_cost": 1e-07, - "max_tokens": 65536, "max_output_tokens": 65536, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77610,7 +74680,6 @@ "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 3e-08, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77627,7 +74696,6 @@ "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77644,7 +74712,6 @@ "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77662,7 +74729,6 @@ "output_cost_per_token": 1e-05, "cache_read_input_token_cost": 2.0000000000000002e-07, "cache_creation_input_token_cost": 2e-06, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77680,7 +74746,6 @@ "output_cost_per_token": 1.5e-05, "cache_read_input_token_cost": 3e-07, "cache_creation_input_token_cost": 3e-06, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77697,7 +74762,6 @@ "input_cost_per_token": 9.499999999999999e-07, "output_cost_per_token": 4e-06, "cache_read_input_token_cost": 1.9e-07, - "max_tokens": 262144, "max_output_tokens": 262144, "max_input_tokens": 262144, "supports_function_calling": true, @@ -77714,7 +74778,6 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77731,7 +74794,6 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-09, - "max_tokens": 384000, "max_output_tokens": 384000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77748,7 +74810,6 @@ "input_cost_per_token": 1.32e-06, "output_cost_per_token": 3.96e-06, "cache_read_input_token_cost": 4.4e-08, - "max_tokens": 384000, "max_output_tokens": 384000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77765,7 +74826,6 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, "cache_read_input_token_cost": 2.8e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77782,7 +74842,6 @@ "input_cost_per_token": 4.35e-07, "output_cost_per_token": 8.7e-07, "cache_read_input_token_cost": 3.625e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77799,7 +74858,6 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, "cache_read_input_token_cost": 2.8e-09, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77816,7 +74874,6 @@ "input_cost_per_token": 4.35e-07, "output_cost_per_token": 8.7e-07, "cache_read_input_token_cost": 3.625e-09, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77833,7 +74890,6 @@ "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, "cache_read_input_token_cost": 1.5e-08, - "max_tokens": 512000, "max_output_tokens": 512000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77850,7 +74906,6 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-08, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77867,7 +74922,6 @@ "input_cost_per_token": 1.0000000000000001e-07, "output_cost_per_token": 2.0000000000000002e-07, "cache_read_input_token_cost": 2e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77884,7 +74938,6 @@ "input_cost_per_token": 1.0000000000000001e-07, "output_cost_per_token": 2.0000000000000002e-07, "cache_read_input_token_cost": 2e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77902,7 +74955,6 @@ "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 2.5e-07, "cache_creation_input_token_cost": 2.5e-06, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77920,7 +74972,6 @@ "output_cost_per_token": 4.6999999999999995e-07, "cache_read_input_token_cost": 1.6e-08, "cache_creation_input_token_cost": 2.0000000000000002e-07, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77938,7 +74989,6 @@ "output_cost_per_token": 1.6000000000000001e-06, "cache_read_input_token_cost": 4e-08, "cache_creation_input_token_cost": 5e-07, - "max_tokens": 65536, "max_output_tokens": 65536, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77977,7 +75027,6 @@ "input_cost_per_token": 8.34e-07, "output_cost_per_token": 2.501e-06, "cache_read_input_token_cost": 4.2000000000000006e-08, - "max_tokens": 64000, "max_output_tokens": 64000, "max_input_tokens": 1024000, "supports_function_calling": true, @@ -77994,7 +75043,6 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 5.8e-07, "cache_read_input_token_cost": 3.5e-08, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 192000, "supports_function_calling": true, @@ -78011,7 +75059,6 @@ "input_cost_per_token": 0, "output_cost_per_token": 0, "cache_read_input_token_cost": 0, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -78028,7 +75075,6 @@ "input_cost_per_token": 1e-06, "output_cost_per_token": 2.7e-06, "cache_read_input_token_cost": 1.0000000000000001e-07, - "max_tokens": 65536, "max_output_tokens": 65536, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -78045,7 +75091,6 @@ "input_cost_per_token": 0, "output_cost_per_token": 0, "cache_read_input_token_cost": 0, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -78066,7 +75111,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -78111,7 +75155,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -78155,7 +75198,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -78188,7 +75230,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -78238,7 +75279,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78275,7 +75315,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78318,7 +75357,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78362,7 +75400,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78407,7 +75444,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78452,7 +75488,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1e-05, @@ -78497,7 +75532,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -78532,7 +75566,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -78575,7 +75608,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -78619,7 +75651,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -78652,5 +75683,327 @@ "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing", "supports_web_search": true + }, + "gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_12hr": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_above_12hr_batches": 1.25e-06, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-06, + "cache_creation_input_token_cost_above_12hr_priority": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-06, + "cache_creation_input_token_cost_batches": 6.25e-07, + "cache_creation_input_token_cost_flex": 6.25e-07, + "cache_creation_input_token_cost_priority": 2.5e-06, + "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 2e-07, + "cache_read_input_token_cost_batches": 2.5e-08, + "cache_read_input_token_cost_flex": 2.5e-08, + "cache_read_input_token_cost_priority": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_272k_tokens": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_flex": 1e-06, + "input_cost_per_token_above_272k_tokens_priority": 4e-06, + "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_priority": 2e-06, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "output_cost_per_token_above_272k_tokens": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-06, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-06, + "output_cost_per_token_above_272k_tokens_priority": 1.5e-05, + "output_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_flex": 2.5e-06, + "output_cost_per_token_priority": 1e-05, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_tool_search": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, + "chatgpt/gpt-6.1-terra": { + "litellm_provider": "chatgpt", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "responses", + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "global.openai.gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, + "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_272k_tokens": 2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "output_cost_per_token_above_272k_tokens": 7.5e-06, + "source": "https://developers.openai.com/api/docs/models/gpt-6.1-terra", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_bedrock_converse_tool_result_images": false, + "supports_bedrock_runtime_chat_completions_response_format": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "us.openai.gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-06, + "cache_read_input_token_cost": 5.5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-07, + "input_cost_per_token": 1.1e-06, + "input_cost_per_token_above_272k_tokens": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-06, + "output_cost_per_token_above_272k_tokens": 8.25e-06, + "source": "https://developers.openai.com/api/docs/models/gpt-6.1-terra", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_bedrock_converse_tool_result_images": false, + "supports_bedrock_runtime_chat_completions_response_format": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "baseten/decision-models/baseten-elephant-9b-latest": { + "input_cost_per_token": 0.0, + "litellm_provider": "baseten", + "max_input_tokens": 16384, + "mode": "decisions", + "output_cost_per_token": 0.0, + "source": "https://inference.baseten.co/v1/models", + "supports_vision": true + }, + "baseten/decision-models/baseten-mammoth-27b-latest": { + "input_cost_per_token": 0.0, + "litellm_provider": "baseten", + "max_input_tokens": 16384, + "mode": "decisions", + "output_cost_per_token": 0.0, + "source": "https://inference.baseten.co/v1/models", + "supports_vision": true + }, + "github_copilot/gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, + "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_272k_tokens": 2e-06, + "litellm_provider": "github_copilot", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 5e-06, + "output_cost_per_token_above_272k_tokens": 7.5e-06, + "source": "https://github.blog/changelog/2026-10-26-gpt-6-1-terra-in-github-copilot", + "supported_endpoints": [ + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "refuseless/glm-5.3-abliterated": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "refuseless", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.25e-06, + "source": "https://refuseless.com/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ] + }, + "refuseless/glm-5.3-flash-abliterated": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "refuseless", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://refuseless.com/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ] + }, + "refuseless/deepseek-v4.1-flash-abliterated": { + "cache_read_input_token_cost": 4.5e-08, + "input_cost_per_token": 2.2e-07, + "litellm_provider": "refuseless", + "max_input_tokens": 1048576, + "max_output_tokens": 130000, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "source": "https://refuseless.com/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ] + }, + "refuseless/deepseek-v4.1-flash-fast": { + "cache_read_input_token_cost": 3e-09, + "input_cost_per_token": 5e-08, + "litellm_provider": "refuseless", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://refuseless.com/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ] + }, + "refuseless/minimax-m3.1-flash": { + "cache_read_input_token_cost": 2e-09, + "input_cost_per_token": 5e-08, + "litellm_provider": "refuseless", + "max_input_tokens": 1048576, + "max_output_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://refuseless.com/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ] } } diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index f95b553c2b4..49b667af819 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -32,6 +32,23 @@ } }, "providers": { + "refuseless": { + "display_name": "Refuseless (`refuseless`)", + "url": "https://docs.litellm.ai/docs/providers/refuseless", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "a2a": { "display_name": "A2A (Agent-to-Agent) (`a2a`)", "url": "https://docs.litellm.ai/docs/providers/a2a", diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 824633bdf5a..f883031a529 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -27,6 +27,7 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import ( get_passthrough_resource_metadata_url, get_passthrough_www_authenticate, get_request_base_url, + get_route_relative_request_path, well_known_root_suffix, ) from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( @@ -387,7 +388,7 @@ def _gateway_dcr_challenge( """The RFC 9728 challenge pointing the client at the protected-resource metadata matching the scope it requested: the per-server document (same URL spelling the request arrived on) when the single target advertises gateway-owned sign-in, - else the gateway's aggregate document. Either way the client discovers the gateway + else a gateway document containing the public resource. Either way the client discovers the gateway as its authorization server and starts the same sign-in flow. ``invalid_token`` adds the RFC 6750 error code for a request that DID @@ -397,10 +398,13 @@ def _gateway_dcr_challenge( client_ip: Final = IPAddressUtils.get_mcp_client_ip(request) target: Final = _gateway_dcr_challenge_target(route, mcp_servers, client_ip) server_scope: Final = gateway_dcr_challenge_scope(route, mcp_servers, client_ip) if target is None else None + public_path: Final = get_route_relative_request_path(request.scope).rstrip("/") + aggregate_resource: Final = public_path == "/mcp" or public_path.startswith("/mcp/") + metadata_suffix: Final = f"{well_known_root_suffix()}/mcp" if aggregate_resource else "" resource_metadata_url: Final = ( get_passthrough_resource_metadata_url(request.scope, target) if target is not None - else f"{get_request_base_url(request)}/.well-known/oauth-protected-resource{well_known_root_suffix()}/mcp" + else f"{get_request_base_url(request)}/.well-known/oauth-protected-resource{metadata_suffix}" ) error_attr: Final = 'error="invalid_token", ' if invalid_token else "" scope_attr: Final = f', scope="{server_scope}"' if server_scope is not None else "" diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index f5c54451006..ca7ad9e3384 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -472,15 +472,15 @@ def redeem_passthrough_authorization_code( return sealed -def _session_cookie_user_id(request: Request) -> str | None: +async def _session_cookie_user_id(request: Request) -> str | None: """The signed-in litellm user for a browser request, or ``None``. Thin wrapper so the aggregate DCR flow's verbs receive the identity as a plain value instead of parsing cookies themselves.""" from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # circular import at module load - user_id_from_session_cookie, + get_authenticated_browser_user_id, ) - return user_id_from_session_cookie(request) + return await get_authenticated_browser_user_id(request) def _redirect_to_litellm_login(request: Request) -> RedirectResponse: @@ -996,17 +996,13 @@ async def _resolve_oauth_authorization_user( enforce_binding: bool, ) -> str | RedirectResponse: """Resolve the authorization subject without replacing denied credentials with cookie grants.""" - from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # proxy import cycle - user_id_from_session_cookie, - ) - use_gateway_credential: Final = enforce_binding and await oauth_authorization_uses_gateway_credential(request) request_user_id: Final = ( await authorize_oauth_credential_request(request, mcp_server.server_id) if use_gateway_credential else None ) if use_gateway_credential and request_user_id is None: return _bridge_access_denied_redirect(redirect_uri, state, mcp_server) - user_id: Final = request_user_id or user_id_from_session_cookie(request) + user_id: Final = request_user_id or await _session_cookie_user_id(request) if user_id is None: return _redirect_to_litellm_login(request) if not await _user_can_reach_mcp_server(user_id, mcp_server.server_id): @@ -1024,7 +1020,7 @@ async def _managed_authorization_scope( request ) session_user: Final = ( - _session_cookie_user_id(request) if authorized_user_id is None and not gateway_credential else None + await _session_cookie_user_id(request) if authorized_user_id is None and not gateway_credential else None ) scope_user_id: Final = authorized_user_id or ( await authorize_oauth_credential_request(request, server.server_id) @@ -2359,7 +2355,7 @@ async def authorize_mcp_session( code_challenge=code_challenge, code_challenge_method=code_challenge_method, response_type=response_type, - session_user_id=_session_cookie_user_id(request), + session_user_id=await _session_cookie_user_id(request), resource=resource, scope=scope, ) @@ -2391,7 +2387,7 @@ async def authorize( code_challenge=code_challenge, code_challenge_method=code_challenge_method, response_type=response_type, - session_user_id=_session_cookie_user_id(request), + session_user_id=await _session_cookie_user_id(request), lookup_consent_teams=lookup_consent_teams, ) return aggregate_authorize( @@ -2402,7 +2398,7 @@ async def authorize( code_challenge=code_challenge, code_challenge_method=code_challenge_method, response_type=response_type, - session_user_id=_session_cookie_user_id(request), + session_user_id=await _session_cookie_user_id(request), resource=resource, scope=scope, ) @@ -2560,7 +2556,7 @@ async def authorize_flow(request: Request, flow: str) -> Response: return await describe_connect_flow( request=request, flow_handle=flow, - session_user_id=_session_cookie_user_id(request), + session_user_id=await _session_cookie_user_id(request), lookup_vendor_credential=_vendor_credential_state, lookup_server_reachability=_user_can_reach_mcp_server, ) @@ -2586,7 +2582,7 @@ async def authorize_complete( return await complete_connect_flow( request=request, flow_handle=flow, - session_user_id=_session_cookie_user_id(request), + session_user_id=await _session_cookie_user_id(request), cache=user_api_key_cache, delivery=delivery, team_id=team_id, diff --git a/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py b/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py index a3e33929f6d..e148e28e826 100644 --- a/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py +++ b/litellm/proxy/_experimental/mcp_server/faults/list_outcomes.py @@ -3,8 +3,8 @@ The aggregate listing deliberately keeps serving the healthy subset when one server fails, but a failed server must contribute a classified outcome instead of silently shrinking the list: an empty contribution with no signal makes a broken upstream indistinguishable from a healthy server with no -tools. Outcomes carry only machine fields (category and status code) so nothing from an upstream -body crosses the trust boundary; classification is total, so any exception out of a server fetch +tools. Outcomes carry machine fields and controlled diagnostics so nothing from an upstream body +crosses the trust boundary; classification is total, so any exception out of a server fetch becomes an outcome, never a second failure. """ @@ -21,8 +21,10 @@ from urllib.request import parse_http_list, parse_keqv_list import anyio import httpx import httpx2 +from mcp import MCPError +from mcp.types import CONNECTION_CLOSED from mcp.types import Tool as MCPTool -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, PositiveInt, TypeAdapter from typing_extensions import assert_never from litellm._logging import verbose_logger @@ -65,6 +67,16 @@ class ServerListFault(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) tag: ListFaultCategory status_code: int | None = None + sse_event_limit_bytes: PositiveInt | None = None + + @property + def public_message(self) -> str | None: + if self.sse_event_limit_bytes is None: + return None + return ( + f"Upstream MCP SSE event exceeded the {self.sse_event_limit_bytes} byte per-event limit. " + "Paginate tools/list responses so each event stays below the limit." + ) ServerOutcome: TypeAlias = ServerListOk | ServerListFault @@ -155,6 +167,18 @@ def raise_classified_list_failure( raise MCPServerListError(classify_list_exception(exc), server_name) from exc +def _sse_event_limit(exc: BaseException) -> int | None: + """The SDK replaces decoder exceptions with CONNECTION_CLOSED and a fixed diagnostic.""" + if not isinstance(exc, (httpx2.SSEError, MCPError)): + return None + if isinstance(exc, MCPError) and exc.error.code != CONNECTION_CLOSED: + return None + prefix: Final = "SSE stream failed: " if isinstance(exc, MCPError) else "" + message: Final = exc.error.message if isinstance(exc, MCPError) else str(exc) + matched: Final = re.fullmatch(prefix + r"Server-sent event exceeded the ([1-9][0-9]{0,18}) byte limit\.", message) + return int(matched[1]) if matched is not None else None + + def classify_list_exception(exc: BaseException) -> ServerListFault: """Classify a per-server listing failure into exactly one outcome. Total: an exception this function cannot recognize is the gateway's own fault (``internal``), never a re-raise.""" @@ -179,6 +203,11 @@ def classify_list_exception(exc: BaseException) -> ServerListFault: response: Final = _find_upstream_response(exc) if response is not None: return ServerListFault(tag="upstream_error", status_code=response.status_code) + sse_event_limit: Final = next( + (limit for error in iter_exception_tree(exc) if (limit := _sse_event_limit(error)) is not None), None + ) + if sse_event_limit is not None: + return ServerListFault(tag="upstream_error", sse_event_limit_bytes=sse_event_limit) if isinstance(exc, (httpx.TimeoutException, httpx2.TimeoutException)): return ServerListFault(tag="timeout") if isinstance(exc, (httpx.TransportError, httpx2.TransportError)): @@ -295,7 +324,7 @@ def listing_failed_outright(outcomes: Mapping[str, ServerOutcome]) -> bool: def outcome_wire_value(outcome: ServerOutcome) -> dict[str, object]: """The client-visible form of one outcome, for the tools/list result ``_meta`` and the REST - response: category plus status code only, never upstream prose or URLs.""" + response: category, status code and controlled diagnostics, never upstream prose or URLs.""" match outcome.tag: case "ok": return {"status": "ok", "tool_count": outcome.tool_count} @@ -303,6 +332,15 @@ def outcome_wire_value(outcome: ServerOutcome) -> dict[str, object]: return { "status": outcome.tag, **({"http_status": outcome.status_code} if outcome.status_code is not None else {}), + **( + { + "reason": "sse_event_too_large", + "limit_bytes": outcome.sse_event_limit_bytes, + "message": outcome.public_message, + } + if outcome.sse_event_limit_bytes is not None + else {} + ), } case _: assert_never(outcome.tag) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index ee063b81ae4..62387b7cc28 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -957,7 +957,7 @@ if MCP_AVAILABLE: status_code=list_fault_http_status(fault), detail={ "error": fault.tag, - "message": f"Failed to list tools from server {get_server_prefix(server)}", + "message": fault.public_message or f"Failed to list tools from server {get_server_prefix(server)}", }, ) from e except Exception as e: diff --git a/litellm/proxy/_experimental/mcp_server/sampling_handler.py b/litellm/proxy/_experimental/mcp_server/sampling_handler.py index e1469f2fff7..d049ada4dfd 100644 --- a/litellm/proxy/_experimental/mcp_server/sampling_handler.py +++ b/litellm/proxy/_experimental/mcp_server/sampling_handler.py @@ -222,7 +222,7 @@ def _select_model_by_priority( input_cost = info.get("input_cost_per_token") or 0.0 output_cost = info.get("output_cost_per_token") or 0.0 total_cost = input_cost + output_cost - max_output = info.get("max_output_tokens") or info.get("max_tokens") or 0 + max_output = info.get("max_output_tokens") or 0 output_tps = info.get("output_tokens_per_second") or 0.0 scored.append( _ScoredModel( diff --git a/litellm/proxy/_experimental/mcp_server/scope_challenge.py b/litellm/proxy/_experimental/mcp_server/scope_challenge.py index c326418bee7..121234e305d 100644 --- a/litellm/proxy/_experimental/mcp_server/scope_challenge.py +++ b/litellm/proxy/_experimental/mcp_server/scope_challenge.py @@ -33,9 +33,8 @@ class OAuthScopeResponse: self._request_path = request_path self._ready = asyncio.Event() self._lock = asyncio.Lock() - self._start: Message | None = None self._challenge: MCPUpstreamAuthError | None = None - self._refused = False + self._closed = False @property def pending(self) -> bool: @@ -50,31 +49,33 @@ class OAuthScopeResponse: self._ready.set() async def __call__(self, message: Message) -> None: - if message["type"] == "http.response.start": - self._start = message - if message["status"] != 200: - self.allow() - return - await self._ready.wait() async with self._lock: - if self._refused: + if self._closed: return - if self._start is not None: - start: Final = self._start - self._start = None - if self._challenge is not None: - error: Final = self._challenge.to_http_exception(self._base_url, self._request_path) - response: Final = JSONResponse( - status_code=error.status_code, content={"detail": error.detail}, headers=error.headers - ) - self._refused = True - await self._send( - {"type": "http.response.start", "status": response.status_code, "headers": response.raw_headers} - ) - await self._send({"type": "http.response.body", "body": bytes(response.body)}) - return - await self._send(start) - await self._send(message) + try: + if message["type"] == "http.response.start": + if message["status"] != 200: + self.allow() + await self._ready.wait() + if self._challenge is not None: + error: Final = self._challenge.to_http_exception(self._base_url, self._request_path) + response: Final = JSONResponse( + status_code=error.status_code, content={"detail": error.detail}, headers=error.headers + ) + self._closed = True + await self._send( + { + "type": "http.response.start", + "status": response.status_code, + "headers": response.raw_headers, + } + ) + await self._send({"type": "http.response.body", "body": bytes(response.body)}) + return + await self._send(message) + except BaseException: + self._closed = True + raise def scope_response_from_request(request: object) -> OAuthScopeResponse | None: diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index aa67d36af4f..4e54d6ac733 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -207,6 +207,11 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( module_path="litellm.proxy.management_endpoints.config_override_endpoints", path_prefixes=("/config_overrides",), ), + LazyFeature( + name="codex_passthrough", + module_path="litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints", + path_prefixes=("/backend-api/codex/",), + ), LazyFeature( name="llm_passthrough", module_path="litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 4e183bf9d5c..166072e9613 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -8673,6 +8673,184 @@ } } }, + "codex_passthrough": { + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail", + "type": "array" + } + }, + "title": "HTTPValidationError", + "type": "object" + }, + "ValidationError": { + "properties": { + "ctx": { + "title": "Context", + "type": "object" + }, + "input": { + "title": "Input" + }, + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location", + "type": "array" + }, + "msg": { + "title": "Message", + "type": "string" + }, + "type": { + "title": "Error Type", + "type": "string" + } + }, + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError", + "type": "object" + } + } + }, + "paths": { + "/backend-api/codex/{endpoint}": { + "get": { + "description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication.", + "operationId": "codex_passthrough_route_backend_api_codex__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "enum": [ + "models", + "responses" + ], + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Codex Passthrough Route", + "tags": [ + "codex_passthrough" + ] + }, + "post": { + "description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication.", + "operationId": "codex_passthrough_route_backend_api_codex__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "enum": [ + "models", + "responses" + ], + "title": "Endpoint", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + }, + "text/event-stream": { + "schema": { + "type": "string" + } + } + }, + "description": "Unmodified Codex response or server-sent events" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Codex Passthrough Route", + "tags": [ + "codex_passthrough" + ] + } + } + } + }, "compliance": { "components": { "schemas": { diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 2b01e02bb06..49d23d82aa9 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -391,6 +391,7 @@ class LiteLLMRoutes(enum.Enum): ] mapped_pass_through_routes = [ + "/backend-api/codex", "/bedrock", "/comprehendmedical", "/azure_speech", @@ -1766,8 +1767,10 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase): max_parallel_requests: int | None = Field( default=None, description="Max concurrent requests allowed for this budget id." ) - tpm_limit: int | None = Field(default=None, description="Max tokens per minute, allowed for this budget id.") - rpm_limit: int | None = Field(default=None, description="Max requests per minute, allowed for this budget id.") + tpm_limit: int | None = Field(default=None, ge=0, description="Max tokens per minute, allowed for this budget id.") + rpm_limit: int | None = Field( + default=None, ge=0, description="Max requests per minute, allowed for this budget id." + ) tpd_limit: int | None = Field( default=None, description="Max tokens per day, charged by batch submissions, allowed for this budget id." ) @@ -1878,6 +1881,8 @@ RouterSettingsDict = Annotated[ class NewTeamRequest(TeamBase): + tpm_limit: int | None = Field(default=None, ge=0) + rpm_limit: int | None = Field(default=None, ge=0) router_settings: RouterSettingsDict | None = None model_aliases: dict | None = None model_max_budget: GenericBudgetConfigType | None = Field( @@ -1957,8 +1962,8 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): team_alias: str | None = None organization_id: str | None = None metadata: dict | None = None - tpm_limit: int | None = None - rpm_limit: int | None = None + tpm_limit: int | None = Field(default=None, ge=0) + rpm_limit: int | None = Field(default=None, ge=0) tpd_limit: int | None = None max_budget: float | None = None soft_budget: float | None = None @@ -3272,7 +3277,7 @@ class LiteLLM_OrganizationTableWithMembers(LiteLLM_OrganizationTable): members: list[LiteLLM_OrganizationMembershipTable] = [] teams: list[LiteLLM_TeamTable] = [] - litellm_budget_table: LiteLLM_BudgetTable | None = None + litellm_budget_table: LiteLLM_BudgetTableFull | None = None created_at: datetime updated_at: datetime diff --git a/litellm/proxy/agent_endpoints/model_list_helpers.py b/litellm/proxy/agent_endpoints/model_list_helpers.py index 4a88644bae5..447b4f52aaf 100644 --- a/litellm/proxy/agent_endpoints/model_list_helpers.py +++ b/litellm/proxy/agent_endpoints/model_list_helpers.py @@ -51,7 +51,7 @@ async def append_agents_to_model_group( async def append_agents_to_model_info( models: list[dict], user_api_key_dict: UserAPIKeyAuth, -) -> list[dict]: +) -> list[dict[str, object]]: """ Append A2A agents to model info list for UI display. diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 79e385c376a..448643ef59b 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -6461,8 +6461,29 @@ def _search_tool_deny_by_default(general_settings: Mapping[str, object]) -> bool return True +def _is_dashboard_session(valid_token: UserAPIKeyAuth | None) -> bool: + return valid_token is not None and valid_token.team_id == UI_TEAM_ID + + +def _is_search_tool_strict_identity(valid_token: UserAPIKeyAuth) -> bool: + if not _is_dashboard_session(valid_token): + return _is_strict_grant_identity(valid_token) + return valid_token.user_role not in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ) + + +def _search_tool_strict_layers( + valid_token: UserAPIKeyAuth, team_object: LiteLLM_TeamTable | None +) -> frozenset[GrantLayer]: + if _is_dashboard_session(valid_token): + return frozenset(("user",)) + return _strict_grant_layers(True, valid_token, team_object) + + def is_search_tool_deny_by_default_applied(valid_token: UserAPIKeyAuth, general_settings: Mapping[str, object]) -> bool: - return _search_tool_deny_by_default(general_settings) and _is_strict_grant_identity(valid_token) + return _search_tool_deny_by_default(general_settings) and _is_search_tool_strict_identity(valid_token) def _search_tool_denied(object_type: GrantLayer, message: str) -> ProxyException: @@ -6506,12 +6527,12 @@ async def resolve_search_tool_grants( load_team_object: TeamObjectLoader, ) -> SearchToolGrants: """ - Without `general_settings.search_tool_deny_by_default`, or for the master key and dashboard sessions, + Without `general_settings.search_tool_deny_by_default`, or for the master key and admin dashboard sessions, the key and team lists only restrict when nonempty. With it, the identities `_strict_grant_layers` names must each list the tool, and a missing record, `null`, `[]`, or a team or user that fails to load grants nothing """ if not is_search_tool_deny_by_default_applied(valid_token, general_settings): - team_object: Final = await load_team_object() + team_object: Final = await load_team_object() if not _is_dashboard_session(valid_token) else None return SearchToolGrants( grants=MappingProxyType( {"key": valid_token.object_permission, "team": team_object.object_permission if team_object else None} @@ -6521,8 +6542,10 @@ async def resolve_search_tool_grants( from litellm.proxy.proxy_server import prisma_client, user_api_key_cache - strict_team_object: Final = await _strict_team_object(load_team_object) - strict_layers: Final = _strict_grant_layers(True, valid_token, strict_team_object) + strict_team_object: Final = ( + await _strict_team_object(load_team_object) if not _is_dashboard_session(valid_token) else None + ) + strict_layers: Final = _search_tool_strict_layers(valid_token, strict_team_object) if prisma_client is None: return SearchToolGrants(grants=MappingProxyType({}), strict_layers=strict_layers) user_object: Final = ( @@ -6607,7 +6630,7 @@ def check_unregistered_search_fallback( """The unregistered provider fallback names no search tool that a grant could list, so deny-by-default denies it""" if not is_search_tool_deny_by_default_applied(valid_token, general_settings): return True - layer: Final = min(_strict_grant_layers(True, valid_token, None), key=("key", "team", "user").index) + layer: Final = min(_search_tool_strict_layers(valid_token, None), key=("key", "team", "user").index) raise _search_tool_denied( layer, "No registered search tool is available and search_tool_deny_by_default is enabled", diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index f6637a1fdf3..2eb25cc6105 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -12,7 +12,7 @@ import litellm from litellm._logging import verbose_proxy_logger, verbose_proxy_stdout_logger from litellm.constants import EMPTY_MAPPING from litellm.integrations.otel.runtime import seed_request_identity -from litellm.litellm_core_utils.core_helpers import is_expected_client_error +from litellm.litellm_core_utils.core_helpers import as_str_mapping, is_expected_client_error from litellm.proxy._types import ( LitellmUserRoles, ModelAccessDeniedProxyException, @@ -139,7 +139,7 @@ class UserAPIKeyAuthExceptionHandler: async def handle_authentication_error( e: Exception, request: Request, - request_data: dict[str, object], + request_data: object, route: str, parent_otel_span: Span | None, api_key: str, @@ -190,6 +190,8 @@ class UserAPIKeyAuthExceptionHandler: request_route=route, ) else: + request_mapping: Final = as_str_mapping(request_data) + request_context: Final[dict[str, object]] = dict(request_mapping) if request_mapping is not None else {} # raise the exception to the caller requester_ip: Final = get_request_ip_address( request=request, @@ -229,7 +231,7 @@ class UserAPIKeyAuthExceptionHandler: # without this the failed trace would carry no team/key attributes. seed_request_identity( user_api_key_dict, - model=request_data.get("model"), + model=request_context.get("model"), ) # Budget checks live in tenant-scoped helpers (key / team / org / tag) @@ -242,14 +244,14 @@ class UserAPIKeyAuthExceptionHandler: resolve_llm_provider_for_rate_limit, ) - budget_model: Final = request_data.get("model") + budget_model: Final = request_context.get("model") _, e.llm_provider = resolve_llm_provider_for_rate_limit( budget_model if isinstance(budget_model, str) else None ) # Allow callbacks to transform the error response transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook( - request_data=_with_client_context(request_data, requester_ip, user_agent), + request_data=_with_client_context(request_context, requester_ip, user_agent), original_exception=e, user_api_key_dict=user_api_key_dict, error_type=ProxyErrorTypes.auth_error, diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 25f900a5f9f..30ede935cfa 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -736,7 +736,9 @@ def is_request_body_safe( def _reject_request_credential_params(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None: if manages_deployments: - reject_server_owned_wif_fields(litellm_params) + reject_server_owned_wif_fields( + MappingProxyType({key: value for key, value in litellm_params.items() if key != "chatgpt_auth_file"}) + ) else: reject_server_owned_wif_params(litellm_params) reject_federated_credential_reference(litellm_params) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index cec176ac559..61cf4d696a1 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -119,6 +119,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( ) from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression +from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import ( route_request, @@ -1416,6 +1417,8 @@ async def open_sse_before_first_byte( if interval is None: return await produce_response + # Share admission state with stream cleanup before the producer copies this context. + get_or_create_request_stash() produce_task: Final = asyncio.ensure_future(produce_response) await asyncio.wait((produce_task,), timeout=interval) if produce_task.done(): diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index a38769ac41b..9f7fed40ab3 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -6,7 +6,7 @@ from typing import Annotated, Any, Final, Literal, Union, get_args, get_origin import orjson from fastapi import Request, UploadFile, status -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from starlette._utils import get_route_path from starlette.requests import HTTPConnection from typing_extensions import NotRequired, ReadOnly, Required, assert_never @@ -162,13 +162,11 @@ def coerce_numeric_form_fields( } -def _parse_binary_body(body: bytes) -> dict: +def _parse_binary_body(body: bytes) -> dict[str, object]: """JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}.""" try: - parsed: Final = _DECODED_JSON.validate_python(orjson.loads(body)) - if isinstance(parsed, dict): - return parsed - except orjson.JSONDecodeError: + return dict(_JSON_REQUEST_BODY.validate_python(orjson.loads(body))) + except (orjson.JSONDecodeError, ValidationError): pass return {} @@ -299,8 +297,9 @@ async def read_request_body(request: Request | None) -> dict: body_str = re.sub(r"(? dict: safe_set_request_parsed_body(request=request, parsed_body=parsed_body) return parsed_body + except ValidationError as e: + from fastapi.routing import APIRoute + + from litellm.proxy.auth.auth_utils import ( + request_dispatched_to_pass_through_endpoint, + request_dispatched_to_provider_pass_through, + ) + from litellm.proxy.lens.buffering import LensRoute + + route: Final[object] = request.scope.get("route") if request is not None else None + if request is not None and ( + request_dispatched_to_pass_through_endpoint(request) + or request_dispatched_to_provider_pass_through(request) + or isinstance(route, LensRoute) + or ( + isinstance(route, APIRoute) + and route.body_field is not None + and not route.body_field.field_info.is_required() + and (await request.body()).strip() == b"null" + ) + ): + return {} + raise ProxyException( + message="Invalid JSON payload: expected an object", + type="invalid_request_error", + param="request_body", + code=status.HTTP_400_BAD_REQUEST, + ) from e except (json.JSONDecodeError, orjson.JSONDecodeError, ProxyException) as e: # Re-raise ProxyException as-is verbose_proxy_logger.error("Invalid JSON payload received: %s", e) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 684f1619512..53c54fe15bf 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -2304,7 +2304,15 @@ async def test_model_connection( ) selected_by_id: Final = deployment_by_id is not None - reject_server_owned_wif_fields(request_litellm_params) + reject_server_owned_wif_fields( + MappingProxyType( + { + key: value + for key, value in _OBJECT_MAPPING.validate_python(request_litellm_params).items() + if key != "chatgpt_auth_file" + } + ) + ) # Merge: config params (from proxy config) as base, request params override litellm_params = { **_config_base_for_health_check( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8dc7501bc3b..5f1948d1c1c 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -99,6 +99,8 @@ _CREDENTIAL_HEADER_NAMES: Final = SpecialHeaders.litellm_credential_header_names {"cookie", "proxy-authorization"} ) _TRANSPORT_ONLY_CREDENTIAL_KEYS: Final = frozenset({"provider_specific_header", "headers", "api_key"}) +_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object]) +_REQUEST_ENTRIES: Final = TypeAdapter(tuple[object, ...]) # Matches any header of the form x--session-id (case-insensitive). # Excludes the two explicit litellm headers which are handled with higher priority. @@ -289,6 +291,9 @@ _DESTINATION_ONLY_CALLBACK_VARS: Final[frozenset[str]] = frozenset({CAPTURE_MESS _CALLBACK_CREDENTIAL_KEYS: Final = frozenset(StandardCallbackDynamicParams.__annotations__) | frozenset( {TRUSTED_CALLBACK_VARS_FIELD} ) +_HEADERS_THAT_CANNOT_CHANGE_PROVIDER_AUTH: Final = frozenset( + {*(f"x-litellm-{key}" for key in StandardLoggingUserAPIKeyMetadata.__annotations__), *ANTHROPIC_API_HEADERS} +) service_logger_obj: Final = ServiceLogging() # used for tracking latency on OTEL # Bounded dedup for stale-alias warnings (FIFO eviction when over cap). _MAX_STALE_ALIAS_WARNING_KEYS: Final = 10_000 @@ -1417,13 +1422,14 @@ def clean_headers( Cleaned headers dict """ from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key + from litellm.llms.chatgpt.common_utils import is_chatgpt_oauth_key clean_headers: Final = {} litellm_key_lower: Final = litellm_key_header_name.lower() if litellm_key_header_name is not None else None for header, value in headers.items(): header_lower = header.lower() - if header_lower == "authorization" and is_anthropic_oauth_key(value): + if header_lower == "authorization" and (is_anthropic_oauth_key(value) or is_chatgpt_oauth_key(value)): if authenticated_with_header is None or authenticated_with_header.lower() != "authorization": clean_headers[header] = value continue @@ -1450,9 +1456,47 @@ def clean_headers( return clean_headers +def _request_fields(value: object) -> Mapping[str, object]: + try: + return _REQUEST_FIELDS.validate_python(value) + except PydanticValidationError: + return MappingProxyType({}) + + +def _request_entries(value: object) -> tuple[object, ...]: + if value is None: + return () + try: + return _REQUEST_ENTRIES.validate_python(value) + except PydanticValidationError: + return (value,) + + +def _carries_a_caller_header(headers: object) -> bool: + return any(name.lower() not in _HEADERS_THAT_CANNOT_CHANGE_PROVIDER_AUTH for name in _request_fields(headers)) + + +def _caller_credential_fields(data: Mapping[str, object]) -> tuple[str, ...]: + """Names of the request fields through which the caller can change how the provider authenticates the call: + ``api_key``, ``organization``, and ``headers``, ``extra_headers`` or a ``provider_specific_header`` entry's + ``extra_headers`` that carry any header besides the key and user details the proxy adds under + ``add_user_information_to_llm_headers`` and the ``anthropic-version`` and ``anthropic-beta`` headers.""" + carried: Final = { + "api_key": "api_key" in data, + "extra_headers": _carries_a_caller_header(data.get("extra_headers")), + "headers": _carries_a_caller_header(data.get("headers")), + "organization": "organization" in data, + "provider_specific_header": any( + _carries_a_caller_header(_request_fields(entry).get("extra_headers")) + for entry in _request_entries(data.get("provider_specific_header")) + ), + } + return tuple(name for name, sent in sorted(carried.items()) if sent) + + def _is_credential_header(header: str) -> bool: """Whether `header` carries a caller credential rather than request context.""" - return header.lower() in _CREDENTIAL_HEADER_NAMES + return header.lower().removeprefix("x-pass-") in _CREDENTIAL_HEADER_NAMES def redact_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]: @@ -2604,6 +2648,8 @@ async def add_litellm_data_to_request( data[_metadata_variable_name]["used_client_oauth_token"] = add_provider_specific_headers_to_request( data=data, headers=_headers ) + caller_fields: Final = _caller_credential_fields(_REQUEST_FIELDS.validate_python(data)) + data["proxy_server_request"]["caller_credential_fields"] = caller_fields # rebind-ok: data is an out-param ## Cache Controls cache_control_header: Final = _headers.get("Cache-Control", None) @@ -4282,6 +4328,7 @@ _NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join( (LlmProviders.AZURE_AI.value, LlmProviders.DATABRICKS.value) ) _ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS)) +_CHATGPT_OAUTH_CREDENTIAL_PROVIDERS: Final = LlmProviders.CHATGPT.value def add_provider_specific_headers_to_request( @@ -4289,6 +4336,7 @@ def add_provider_specific_headers_to_request( headers: Mapping[str, str], ) -> bool: from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key + from litellm.llms.chatgpt.common_utils import select_chatgpt_client_credential_headers anthropic_api_headers: Final = {header: headers[header] for header in ANTHROPIC_API_HEADERS if header in headers} anthropic_oauth_credential_headers: Final = { @@ -4296,6 +4344,8 @@ def add_provider_specific_headers_to_request( for header, value in headers.items() if header.lower() == "authorization" and is_anthropic_oauth_key(value) } + chatgpt_oauth_credential_headers: Final = dict(select_chatgpt_client_credential_headers(headers)) + typesafe_danger_zone_headers: Final = { header: value for header, value in headers.items() @@ -4320,13 +4370,17 @@ def add_provider_specific_headers_to_request( ProviderSpecificHeader( custom_llm_provider=LlmProviders.TYPESAFE.value, extra_headers=typesafe_danger_zone_headers ), + ProviderSpecificHeader( + custom_llm_provider=_CHATGPT_OAUTH_CREDENTIAL_PROVIDERS, + extra_headers=chatgpt_oauth_credential_headers, + ), ) if scoped_header["extra_headers"] ] if scoped_headers: data["provider_specific_header"] = scoped_headers[0] if len(scoped_headers) == 1 else scoped_headers - return bool(anthropic_oauth_credential_headers) + return bool(anthropic_oauth_credential_headers or chatgpt_oauth_credential_headers) def _add_otel_traceparent_to_data(data: dict, request: Request): diff --git a/litellm/proxy/management_endpoints/common_utils.py b/litellm/proxy/management_endpoints/common_utils.py index adf89c29279..983c3d24513 100644 --- a/litellm/proxy/management_endpoints/common_utils.py +++ b/litellm/proxy/management_endpoints/common_utils.py @@ -569,8 +569,8 @@ _TEAM_MEMBER_BUDGET_LIMIT_FIELDS: Final = ( "temp_budget_expiry", ) -_TEMP_BUDGET_FIELDS: Final = frozenset({"temp_budget_increase", "temp_budget_expiry"}) +_BUDGET_COLUMNS: Final = TypeAdapter(dict[str, object]) MEMBER_BUDGET_PATCH_FIELDS: Final = MappingProxyType( { @@ -653,8 +653,8 @@ async def upsert_budget_and_membership( ``shared_budget_ids`` extends that protection to any other row more than one membership points at, which a caller patching several members at once has already counted; a row listed there is cloned rather than written in place. - A patch that only touches the temporary budget pair never copies permanent - limits into a new row, so the member keeps inheriting the live team default. + A patch whose every field already matches the shared row changes nothing and + is a no-op rather than a clone. """ ignore_default_model_budget_clear: Final = ( existing_budget_id is not None @@ -685,7 +685,6 @@ async def upsert_budget_and_membership( is_shared_default: Final = existing_budget_id is not None and ( existing_budget_id == team_default_budget_id or existing_budget_id in (shared_budget_ids or frozenset()) ) - temp_only: Final = frozenset(write_data) <= _TEMP_BUDGET_FIELDS async def _disconnect(): await tx.litellm_teammembership.update( @@ -707,11 +706,15 @@ async def upsert_budget_and_membership( return source_row: Final = ( - await tx.litellm_budgettable.find_unique(where={"budget_id": existing_budget_id}) - if is_shared_default and not temp_only - else None + await tx.litellm_budgettable.find_unique(where={"budget_id": existing_budget_id}) if is_shared_default else None ) - source: Final[Mapping[str, object]] = source_row.model_dump() if source_row is not None else MappingProxyType({}) + source: Final[Mapping[str, object]] = ( + MappingProxyType(_BUDGET_COLUMNS.validate_python(source_row.model_dump())) + if source_row is not None + else MappingProxyType({}) + ) + if is_shared_default and all(source.get(field) == value for field, value in effective_patch.items()): + return create_data: Final[dict[str, object]] = { # mutable-ok: Prisma create payloads are dict-shaped "created_by": user_api_key_dict.user_id or "", @@ -740,7 +743,7 @@ async def upsert_budget_and_membership( create_data.pop("budget_reset_at", None) if not _has_meaningful_budget_limit(create_data): - if existing_budget_id is not None and not temp_only: + if existing_budget_id is not None: await _disconnect() return diff --git a/litellm/proxy/management_endpoints/key_budget_change.py b/litellm/proxy/management_endpoints/key_budget_change.py index 3143ef53d76..197a13cdb13 100644 --- a/litellm/proxy/management_endpoints/key_budget_change.py +++ b/litellm/proxy/management_endpoints/key_budget_change.py @@ -9,6 +9,7 @@ from typing import Final, Literal, TypeAlias from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.duration_parser import normalize_duration from litellm.models.team import BudgetLimitEntry from litellm.models.verification_token import LiteLLM_VerificationToken from litellm.proxy._types import UpdateKeyRequest @@ -99,6 +100,30 @@ def _budget_limits_change(data: UpdateKeyRequest, existing: LiteLLM_Verification return "tightened" if kept_or_lowered else "loosened" +def _reset_schedule(budget_duration: str | None) -> str | None: + return None if budget_duration is None else normalize_duration(budget_duration) + + +def _changes_budget_duration(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> bool: + return "budget_duration" in data.model_fields_set and _reset_schedule(data.budget_duration) != _reset_schedule( + existing.budget_duration + ) + + +def _changes_budget_id(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> bool: + return "budget_id" in data.model_fields_set and data.budget_id != existing.budget_id + + +def _budget_duration_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> _FieldChange: + """ + Any new `budget_duration` restarts the reset schedule from now, so only clearing it tightens the + stored `max_budget`. A key with no `max_budget` has no cap for the schedule to reset. + """ + if existing.max_budget is None or not _changes_budget_duration(data, existing): + return "unchanged" + return "tightened" if data.budget_duration is None else "loosened" + + @dataclass(frozen=True, slots=True) class EffectiveKeyBudget: max_budget: float | None @@ -122,12 +147,23 @@ def effective_key_budget(data: UpdateKeyRequest, existing: LiteLLM_VerificationT def classify_key_budget_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> KeyBudgetChange: - overrides_linked_budget: Final = ( - existing.budget_id is not None and _max_budget_change(data, existing) != "unchanged" + overrides_linked_budget: Final = existing.budget_id is not None and ( + _max_budget_change(data, existing) != "unchanged" or _changes_budget_duration(data, existing) ) - if data.spend is not None or "soft_budget" in data.model_fields_set or overrides_linked_budget: + if ( + data.spend is not None + or "soft_budget" in data.model_fields_set + or overrides_linked_budget + or _changes_budget_id(data, existing) + ): return KeyBudgetAdminOnly() - changes: Final = frozenset((_max_budget_change(data, existing), _budget_limits_change(data, existing))) + changes: Final = frozenset( + ( + _max_budget_change(data, existing), + _budget_limits_change(data, existing), + _budget_duration_change(data, existing), + ) + ) if "loosened" in changes: return KeyBudgetLoosened() if "tightened" in changes: diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 11f616a5e18..e625ea6f9f2 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -145,6 +145,7 @@ from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.bedrock import AwsSessionTag from litellm.types.proxy.management_endpoints.model_management_endpoints import ( AutoRouterClassifierDefaultPromptResponse, + PatchModelRequest, UpdateUsefulLinksRequest, ) from litellm.types.router import ( @@ -160,6 +161,7 @@ from litellm.types.utils import ( OIDC_SECRET_REFERENCE_PREFIX, echoed_cost_map_fields, echoed_cost_map_pricing_fields, + is_server_derived_pricing_key, without_server_derived_pricing, ) from litellm.utils import get_utc_datetime @@ -995,6 +997,7 @@ def _ptu_zeroed_pricing( def _ptu_pricing_delta( *, stored_model_info: Mapping[str, object], + stored_litellm_params: Mapping[str, object], model_info: Mapping[str, object], litellm_params: Mapping[str, object], patch: updateDeployment, @@ -1014,6 +1017,24 @@ def _ptu_pricing_delta( supplied: Final = patch.litellm_params.model_dump(exclude_none=True) if patch.litellm_params else _EMPTY_MODEL_INFO zeroed: Final = _ptu_zeroed_pricing(model_info=model_info, litellm_params=litellm_params, supplied=supplied) if zeroed: + retains_ptu: Final = all(stored_model_info.get(field) is not None for field in _PTU_PRICED_PAIR) + protected_pricing: Final = frozenset( + field + for field in _CUSTOM_PRICING_FIELDS + if retains_ptu + and (stored_litellm_params.get(field) is not None or stored_model_info.get(field) is not None) + ).union(zeroed) + removed_ptu_pricing: Final = tuple( + field for field in _param_keys_to_remove(patch) if field in protected_pricing + ) + if removed_ptu_pricing: + raise HTTPException( + status_code=400, + detail=( + f"PTU billing requires {', '.join(removed_ptu_pricing)}, so these pricing parameters cannot be removed. " + "Keep them when enabling PTU, or clear ptu_count and cost_per_ptu_per_hour together to disable it." + ), + ) return zeroed, frozenset() was_ptu: Final = any(stored_model_info.get(field) is not None for field in _PTU_PRICED_PAIR) if not was_ptu or not _explicitly_cleared_ptu_fields(patch.model_info) & _PTU_PRICED_PAIR: @@ -1069,6 +1090,10 @@ def _loaded_catalog_entry( return loaded_catalog().get(catalog_key, MappingProxyType({})) +def _param_keys_to_remove(patch: updateDeployment) -> tuple[str, ...]: + return patch.litellm_params_to_remove if isinstance(patch, PatchModelRequest) else () + + def update_db_model( db_model: Deployment, updated_patch: updateDeployment, @@ -1077,7 +1102,8 @@ def update_db_model( if updated_patch.model_info is not None: _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) merged_model_name: Final = updated_patch.model_name or db_model.model_name - merged_litellm_params: Final = db_model.litellm_params.model_dump(exclude_none=True) + stored_litellm_params: Final = db_model.litellm_params.model_dump(exclude_none=True, exclude_unset=True) + merged_litellm_params: Final = dict(stored_litellm_params) stored_model_info: Final = _dump_without_none(db_model.model_info) echoed_pricing: Final = echoed_cost_map_pricing_fields(stored_model_info) merged_model_info: Final[dict[str, object]] = { @@ -1093,7 +1119,7 @@ def update_db_model( if k == "complexity_router_config" else encrypt_value_helper(v) ) - for k, v in updated_patch.litellm_params.model_dump(exclude_none=True).items() + for k, v in updated_patch.litellm_params.model_dump(exclude_none=True, exclude_unset=True).items() } merged_litellm_params.update(encrypted_params) @@ -1144,11 +1170,17 @@ def update_db_model( for field in _explicitly_cleared_ptu_fields(updated_patch.model_info): merged_model_info.pop(field, None) + for field in _param_keys_to_remove(updated_patch): + merged_litellm_params.pop(field, None) + if is_server_derived_pricing_key(field): + merged_model_info.pop(field, None) + _validate_ptu_model_info(merged_model_info) ptu_pricing, ptu_released = _ptu_pricing_delta( stored_model_info=db_model.model_info.model_dump(exclude_none=True) if db_model.model_info else _EMPTY_MODEL_INFO, + stored_litellm_params=stored_litellm_params, model_info=merged_model_info, litellm_params=merged_litellm_params, patch=updated_patch, @@ -1255,7 +1287,7 @@ async def get_kubernetes_pods( ) async def patch_model( model_id: str, # Get model_id from path parameter - patch_data: updateDeployment, # Create a specific schema for PATCH operations + patch_data: PatchModelRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -1319,18 +1351,34 @@ async def patch_model( param=None, ) + removals: Final = _param_keys_to_remove(patch_data) + incoming_params: Final = ( + updateLiteLLMParams.model_validate( + { + **(patch_data.litellm_params.model_dump(exclude_unset=True) if patch_data.litellm_params else {}), + **dict.fromkeys(removals), + } + ) + if removals + else patch_data.litellm_params + ) + remaining_params: Final = db_model.litellm_params.model_copy(update=dict.fromkeys(removals)) write_authorization: Final = await ModelManagementAuthChecks.can_user_make_model_call( model_params=db_model, user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, premium_user=premium_user, - incoming_params=submitted_litellm_params(patch_data.litellm_params), + incoming_params=submitted_litellm_params(incoming_params), member_operation="update", incoming_model_params=patch_data, ) member_write: Final = write_authorization if isinstance(write_authorization, MemberAutoRouterWrite) else None - member_marker: Final = _member_auto_router_marker_for_update( - incoming_params=patch_data.litellm_params, existing=db_model, member_write=member_write + member_marker: Final = ( + False + if db_model.model_info.member_auto_router and STRATEGY_ROUTER_PARAM_FIELDS.intersection(removals) + else _member_auto_router_marker_for_update( + incoming_params=incoming_params, existing=db_model, member_write=member_write + ) ) marker_info: Final = ( ModelInfo(id=db_model.model_info.id) @@ -1367,27 +1415,26 @@ async def patch_model( ) ModelManagementAuthChecks.can_user_attach_credential( - litellm_params=patch_data.litellm_params, + litellm_params=incoming_params, user_api_key_dict=user_api_key_dict, existing_litellm_params=db_model.litellm_params, null_detaches=True, ) - await _raise_on_invalid_credential_name(patch_data.litellm_params, db_model.litellm_params, prisma_client) + await _raise_on_invalid_credential_name(incoming_params, db_model.litellm_params, prisma_client) ModelManagementAuthChecks.can_user_set_aws_session_tags( - litellm_params=patch_data.litellm_params, + litellm_params=incoming_params, user_api_key_dict=user_api_key_dict, existing_litellm_params=db_model.litellm_params, + null_detaches="aws_session_tags" in removals, ) _raise_on_strategy_router_write_violation( - incoming_params=patch_data.litellm_params, - existing_params=db_model.litellm_params, + incoming_params=incoming_params, + existing_params=remaining_params, ) - effective_params: Final = _effective_complexity_router_params( - patch_data.litellm_params, db_model.litellm_params - ) + effective_params: Final = _effective_complexity_router_params(incoming_params, remaining_params) requested_model_name: Final = patch_data.model_name stored_model_name: str | None = None @@ -1818,20 +1865,20 @@ async def _update_team_model_in_db( db_model=db_model, ) - # Ensure model_info exists and set team_public_model_name - if patch_data.model_info is None: - from litellm.types.router import ModelInfo - - patch_data.model_info = ModelInfo() - patch_data.model_info.team_public_model_name = public_model_name - # Check if team assignment is new or changed db_team_id: Final = db_model.model_info.team_id if db_model.model_info else None is_new_team_assignment: Final = db_team_id != patch_team_id # Team rows keep their internal UUID-based model_name; the public name lives in model_info - patch_data.model_name = f"model_name_{patch_team_id}_{uuid.uuid4()}" if is_new_team_assignment else None - row: Final = await write_row(update_db_model(db_model=db_model, updated_patch=patch_data)) + team_patch: Final = patch_data.model_copy( + update={ + "model_name": f"model_name_{patch_team_id}_{uuid.uuid4()}" if is_new_team_assignment else None, + "model_info": (patch_data.model_info or ModelInfo()).model_copy( + update={"team_public_model_name": public_model_name} + ), + } + ) + row: Final = await write_row(update_db_model(db_model=db_model, updated_patch=team_patch)) if is_new_team_assignment: await _setup_new_team_model_assignment( @@ -2228,14 +2275,16 @@ class ModelManagementAuthChecks: litellm_params: GenericLiteLLMParams | None, user_api_key_dict: UserAPIKeyAuth, existing_litellm_params: GenericLiteLLMParams | None = None, + *, + null_detaches: bool = False, ) -> Literal[True]: - if litellm_params is None or litellm_params.aws_session_tags is None: + if litellm_params is None or (litellm_params.aws_session_tags is None and not null_detaches): return True if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN: return True existing_tags: Final = existing_litellm_params.aws_session_tags if existing_litellm_params is not None else None if existing_tags is not None and _canonical_session_tags(existing_tags) == _canonical_session_tags( - litellm_params.aws_session_tags + litellm_params.aws_session_tags or () ): return True raise ProxyException( diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index d7a44cdd0b8..e1d19d8efae 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -504,9 +504,8 @@ async def new_organization( _budget: Final = await _table(BudgetRepository(prisma_client)).create( data={ - **new_budget, + **build_budget_write_data(new_budget, user_api_key_dict.user_id or litellm_proxy_admin_name), "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, - "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, } ) diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py new file mode 100644 index 00000000000..3087faa9f67 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -0,0 +1,87 @@ +import hmac +from types import MappingProxyType +from typing import Annotated, Final, Literal + +from fastapi import APIRouter, Depends, HTTPException, Request, Response + +from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request + +router: Final = APIRouter() +_HEADERS: Final = frozenset( + { + "authorization", + "chatgpt-account-id", + "content-type", + "accept", + "user-agent", + "originator", + "openai-beta", + "session-id", + "thread-id", + "session_id", + "conversation_id", + "x-codex-turn-metadata", + "x-codex-turn-state", + } +) + + +@router.get("/backend-api/codex/{endpoint}", tags=["ChatGPT Pass-through", "pass-through"]) +@router.post( + "/backend-api/codex/{endpoint}", + tags=["ChatGPT Pass-through", "pass-through"], + openapi_extra={ + "requestBody": { + "required": True, + "content": {"application/json": {"schema": {"type": "object", "additionalProperties": True}}}, + } + }, + responses={ + 200: { + "description": "Unmodified Codex response or server-sent events", + "content": { + "application/json": {"schema": {"type": "object", "additionalProperties": True}}, + "text/event-stream": {"schema": {"type": "string"}}, + }, + } + }, +) +async def codex_passthrough_route( + endpoint: Literal["models", "responses"], + request: Request, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + """Forward a client's ChatGPT subscription with separate LiteLLM authentication.""" + expected_method: Final = "GET" if endpoint == "models" else "POST" + if request.method != expected_method: + raise HTTPException(status_code=405, detail="Method not allowed", headers={"Allow": expected_method}) + + authorization: Final = request.headers.get("authorization", "") + scheme, _, token = authorization.partition(" ") + gateway_key: Final = request.headers.get("x-litellm-api-key", "") + gateway_parts: Final = gateway_key.split() + gateway_token: Final = ( + gateway_parts[-1] + if len(gateway_parts) == 1 or (len(gateway_parts) == 2 and gateway_parts[0].lower() == "bearer") + else "" + ) + if scheme.lower() != "bearer" or not token.strip() or not gateway_token: + raise HTTPException( + status_code=401, + detail="Send the ChatGPT bearer in Authorization and the LiteLLM key in x-litellm-api-key", + ) + if hmac.compare_digest(token.strip().encode(), gateway_token.encode()): + raise HTTPException(status_code=401, detail="ChatGPT and LiteLLM credentials must be separate") + + headers: Final = MappingProxyType({name: value for name, value in request.headers.items() if name in _HEADERS}) + return await pass_through_request( + request=request, + target=f"{CHATGPT_API_BASE}/{endpoint}", + custom_headers=dict(headers), + user_api_key_dict=user_api_key_dict, + custom_llm_provider="chatgpt", + upstream_headers_filter=lambda _: headers, + ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py index c763b5dc26e..c410150df70 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py @@ -10,6 +10,7 @@ from typing import Final, Literal, TypeAlias from urllib.parse import urlparse import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_proxy_logger @@ -95,6 +96,8 @@ def is_openai_compatible_url(url_route: str | None) -> bool: hostname: Final = parsed_url.hostname if not hostname: return False + if hostname == "chatgpt.com" and parsed_url.path.startswith("/backend-api/codex/"): + return True if _hostname_matches(hostname, _OPENAI_HOSTNAMES): return True if _hostname_matches(hostname, _AZURE_OPENAI_HOSTNAMES): @@ -198,7 +201,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): if not url_route: return False parsed_url: Final = urlparse(url_route) - return _is_openai_compatible_host(parsed_url.hostname) and ( + return is_openai_compatible_url(url_route) and ( "/v1/responses" in parsed_url.path or "/responses" in parsed_url.path ) @@ -424,7 +427,9 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler): ) = None handler_instance: Final = OpenAIPassthroughLoggingHandler() - custom_llm_provider: Final = kwargs.get("custom_llm_provider", "openai") + custom_llm_provider: Final = TypeAdapter(str).validate_python( + kwargs.get("custom_llm_provider", logging_obj.model_call_details.get("custom_llm_provider", "openai")) + ) if is_chat_completions: # Handle chat completions with existing logic diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index e1e8864678e..fc0e5be3f4c 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -118,6 +118,7 @@ from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy modul _key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy _strip_client_pricing_overrides, # pyright: ignore[reportPrivateUsage] # sanitize before trusted hooks add guardrail costs get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above + redact_credential_headers, ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.route_priority import ( @@ -409,6 +410,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): @staticmethod def get_endpoint_type(url: str, custom_llm_provider: str | None = None) -> EndpointType: parsed_url: Final = urlparse(url) + if custom_llm_provider == "chatgpt": + return EndpointType.OPENAI if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith( "/v1/systemone" ): @@ -723,7 +726,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): "url": str(request.url), "method": request.method, "body": copy.copy(_parsed_body), # use copy instead of deepcopy - "headers": request.headers, + "headers": redact_credential_headers(request.headers), }, }, "call_type": "pass_through_endpoint", @@ -1490,7 +1493,7 @@ async def pass_through_request( additional_args={ "complete_input_dict": _parsed_body, "api_base": str(logging_url), - "headers": upstream_headers, + "headers": redact_credential_headers(upstream_headers), }, ) stream = HttpPassThroughEndpointHelpers.update_stream_param_based_on_request_body( @@ -1561,7 +1564,7 @@ async def pass_through_request( data=_parsed_body or {}, user_api_key_dict=user_api_key_dict, response=relay_response, - request_headers=dict(request.headers), + request_headers=dict(redact_credential_headers(request.headers)), ) if callback_headers: _response_headers.update(callback_headers) @@ -1653,7 +1656,7 @@ async def pass_through_request( data=_parsed_body or {}, user_api_key_dict=user_api_key_dict, response=detected_relay_response, - request_headers=dict(request.headers), + request_headers=dict(redact_credential_headers(request.headers)), ) if callback_headers: _response_headers.update(callback_headers) @@ -1703,7 +1706,7 @@ async def pass_through_request( data=_parsed_body or {}, user_api_key_dict=user_api_key_dict, response=response, - request_headers=dict(request.headers), + request_headers=dict(redact_credential_headers(request.headers)), ) if relay_callback_headers: relay_custom_headers.update(relay_callback_headers) @@ -1878,7 +1881,7 @@ async def pass_through_request( data=_parsed_body or {}, user_api_key_dict=user_api_key_dict, response=response, - request_headers=dict(request.headers), + request_headers=dict(redact_credential_headers(request.headers)), ) if callback_headers: custom_headers.update(callback_headers) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index df948b69a67..ca0f8892b9d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -16760,6 +16760,35 @@ async def _find_model_by_id( return all_models, search_total_count +async def _models_with_stored_param_keys( + models: Sequence[Mapping[str, object]], model_id: str, prisma_client: PrismaClient +) -> Sequence[Mapping[str, object]]: + visible_model: Final = next( + (model for model in models if _with_get(model.get("model_info", {})).get("id") == model_id), None + ) + if visible_model is None: + return models + stored: Final = await ModelRepository(WriterPinnedClient(prisma_client.db)).table.find_unique( + where={"model_id": model_id} + ) + if stored is None: + return models + raw_params: Final = _UNTYPED_VALUE.validate_python(stored.litellm_params) + stored_params: Final = ( + _LITELLM_PARAMS_COPY.validate_json(raw_params) + if isinstance(raw_params, str) + else _LITELLM_PARAMS_COPY.validate_python(raw_params) + ) + visible_params: Final = _LITELLM_PARAMS_COPY.validate_python(visible_model.get("litellm_params", {})) + editor_model: Final = MappingProxyType( + { + **visible_model, + "litellm_params": {key: value for key, value in visible_params.items() if key in stored_params}, + } + ) + return tuple(editor_model if model is visible_model else model for model in models) + + @router.get( "/v2/model/info", tags=["model management"], @@ -16777,6 +16806,10 @@ async def model_info_v2( size: int = Query(50, description="Page size", ge=1), search: str | None = fastapi.Query(None, description="Search model names (case-insensitive partial match)"), modelId: str | None = fastapi.Query(None, description="Search for a specific model by its unique ID"), + use_stored_params: bool = fastapi.Query( + False, + description="With modelId, omit runtime-only litellm_params defaults from a DB-stored model's editor view.", + ), teamId: str | None = fastapi.Query( None, description="Filter models by team ID. Returns models with direct_access=True or teamId in access_via_team_ids", @@ -16828,6 +16861,7 @@ async def model_info_v2( page / size: Pagination controls (defaults: page=1, size=50). search: Case-insensitive partial match on model name or team public name. modelId: Return a single deployment by LiteLLM model id. + use_stored_params: With modelId, return only persisted litellm_params keys, retaining normal redaction. teamId: Filter to models with direct access or team membership for this team id. sortBy / sortOrder: Sort by model_name, created_at, updated_at, costs, or status. blocked: Filter by routing status (false = active, true = paused). @@ -16864,6 +16898,9 @@ async def model_info_v2( """ global llm_model_list, general_settings, user_config_file_path, proxy_config, llm_router + if use_stored_params is True and not isinstance(modelId, str): + raise HTTPException(status_code=400, detail="use_stored_params requires modelId") + # Return empty data array when no models are configured (graceful handling for fresh installs) if llm_router is None or not llm_router.model_list: return { @@ -16999,6 +17036,14 @@ async def model_info_v2( # Translate `model_name` to the public name for team-scoped rows. all_models = [_translate_model_name_for_response(m) for m in all_models] + if use_stored_params is True and isinstance(modelId, str): + return _paginate_models_response( + all_models=await _models_with_stored_param_keys(all_models, modelId, prisma_client), + page=page, + size=size, + total_count=search_total_count, + search=search, + ) return _paginate_models_response( all_models=all_models, page=page, @@ -17512,7 +17557,7 @@ def _filter_v1_model_info_deployments( return [model for model in all_models if _deployment_matches_allowed_model_names(model, allowed_model_names)] -def _translate_model_name_for_response(model: dict) -> dict: +def _translate_model_name_for_response(model: dict) -> dict[str, object]: """For team-scoped DB rows, replace `model_name` with the public name in `model_info.team_public_model_name` before returning. The DB column and the in-memory router index keep the internal mangled name @@ -21795,8 +21840,7 @@ async def toolset_mcp_route(toolset_name: str, request: Request): detail=f"Toolset '{toolset_name}' not found", ) - scope: Final = dict(request.scope) - scope["path"] = "/mcp" + scope: Final = {**request.scope, "_original_path": request.scope.get("path", ""), "path": "/mcp"} token: Final = mcp_active_toolset_id.set(toolset.toolset_id) try: diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 19ee465c7dc..d96f2976508 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -17,6 +17,34 @@ ], "default_model_placeholder": "scaledown/summarize" }, + { + "provider": "ABLITERATION", + "provider_display_name": "Abliteration", + "litellm_provider": "abliteration", + "credential_fields": [ + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.abliteration.ai/v1", + "tooltip": "Optional. Leave blank to use the provider's default endpoint.", + "required": false, + "field_type": "text", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "abliteration/abliterated-model-large-v2" + }, { "provider": "AIML", "provider_display_name": "AI/ML API", @@ -3203,6 +3231,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "REFUSELESS", + "provider_display_name": "Refuseless", + "litellm_provider": "refuseless", + "credential_fields": [ + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.refuseless.com/v1", + "tooltip": "Optional. Leave blank to use the provider's default endpoint.", + "required": false, + "field_type": "text", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "refuseless/glm-5.3-abliterated" + }, { "provider": "REPLICATE", "provider_display_name": "Replicate", diff --git a/litellm/proxy/response_polling/polling_handler.py b/litellm/proxy/response_polling/polling_handler.py index 787f27f56c8..e50587b1a0e 100644 --- a/litellm/proxy/response_polling/polling_handler.py +++ b/litellm/proxy/response_polling/polling_handler.py @@ -308,10 +308,7 @@ def should_use_polling_for_request( # Otherwise, check ALL deployments for this model_name in router elif llm_router is not None: try: - # Get all deployment indices for this model name - indices: Final = llm_router.model_name_to_deployment_indices.get(model, []) - for idx in indices: - deployment_dict = llm_router.model_list[idx] + for deployment_dict in llm_router.deployments_named(model) or (): litellm_params = deployment_dict.get("litellm_params", {}) # Check custom_llm_provider first diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 2aab8deb034..604a7a389e6 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -19,6 +19,7 @@ from pydantic import ConfigDict, Field import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_prompt_caching_savings, generic_cost_per_token, @@ -69,7 +70,9 @@ def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _Model if not model: return None try: - resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + resolved_model, provider = resolve_model_and_provider_for_metadata( + model=model, custom_llm_provider=custom_llm_provider + ) except Exception as e: # noqa: BLE001 # get_llm_provider raises for unroutable names; degrade to an unavailable estimate verbose_proxy_logger.debug( "savings: cannot resolve provider for model=%s custom_llm_provider=%s (%s)", model, custom_llm_provider, e diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 8b00cdf4813..504150b330c 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -10,7 +10,7 @@ from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, foundry_chat_rejects_function_tools_while_reasoning, ) -from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import OpenAIWebSearchOptions from litellm.types.utils import LlmProviders diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 1ff80b0d71d..d993b05f5b7 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -705,10 +705,12 @@ def _api_base_kwarg(kwargs: Mapping[str, object]) -> str | None: return api_base if isinstance(api_base, str) else None -def _dispatched_model_name(model: str, custom_llm_provider: str, api_base: str | None) -> str: - provider_model, _, _, _ = litellm.get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=api_base - ) +def _dispatched_model_name( + model: str, custom_llm_provider: str, api_base: str | None, litellm_params: GenericLiteLLMParams | None = None +) -> str: + from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata + + provider_model, _ = resolve_model_and_provider_for_metadata(model=model, custom_llm_provider=custom_llm_provider) return _strip_responses_routing_prefix(provider_model) @@ -718,6 +720,7 @@ def _will_bridge_to_chat_completions( use_chat_completions_api: bool, model_info: object, api_base: str | None, + litellm_params: GenericLiteLLMParams | None = None, ) -> bool: """``_bridges_to_chat_completions`` for callers running before the provider config is resolved. @@ -734,7 +737,7 @@ def _will_bridge_to_chat_completions( return True return _bridges_to_chat_completions( _resolve_responses_api_provider_config( - _dispatched_model_name(normalized_model[0], custom_llm_provider, api_base), + _dispatched_model_name(normalized_model[0], custom_llm_provider, api_base, litellm_params), custom_llm_provider, model_info, api_base, @@ -750,12 +753,15 @@ def provider_stores_the_conversation(model: str, custom_llm_provider: str | None first MCP round runs as an internal call that writes no spend log at all. """ api_base: Final = _api_base_kwarg(kwargs) + litellm_params: Final = GenericLiteLLMParams(**kwargs) return not _will_bridge_to_chat_completions( model, - custom_llm_provider or litellm.get_llm_provider(model=model, api_base=api_base)[1], + custom_llm_provider + or litellm.get_llm_provider(model=model, api_base=api_base, litellm_params=litellm_params)[1], bool(kwargs.get("use_chat_completions_api")), kwargs.get("model_info"), api_base, + litellm_params, ) @@ -905,7 +911,8 @@ async def aresponses( model=model, api_base=local_vars.get("base_url", None), litellm_params=GenericLiteLLMParams( - **cast("dict[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict + extra_headers=extra_headers, + **cast("dict[str, object]", kwargs), # cast-ok: kwargs is the untyped request dict ), ) # Update local_vars with detected provider (fixes #19782) @@ -936,6 +943,7 @@ async def aresponses( bool(kwargs.get("use_chat_completions_api")), kwargs.get("model_info"), _api_base_kwarg(kwargs), + GenericLiteLLMParams(**kwargs), ), ): _stamp_injection_points_for_the_hook(kwargs, model, custom_llm_provider, input, tools) @@ -967,7 +975,7 @@ async def aresponses( original_model=original_model, swapped_model=model, custom_llm_provider=custom_llm_provider, - kwargs=kwargs, + kwargs={**kwargs, "extra_headers": extra_headers}, prompt_id=prompt_id, ) kwargs.pop("prompt_id", None) @@ -1051,11 +1059,14 @@ def _resolve_prompt_swapped_provider( kwargs: Mapping[str, object], prompt_id: str | None, ) -> str: - swapped_provider: Final = litellm.get_llm_provider(model=swapped_model)[1] + litellm_params: Final = GenericLiteLLMParams(**kwargs) + swapped_provider: Final = litellm.get_llm_provider(model=swapped_model, litellm_params=litellm_params)[1] if kwargs.get("api_key") is None and kwargs.get("api_base") is None: return swapped_provider try: - original_provider: Final = custom_llm_provider or litellm.get_llm_provider(model=original_model)[1] + original_provider: Final = ( + custom_llm_provider or litellm.get_llm_provider(model=original_model, litellm_params=litellm_params)[1] + ) except litellm.BadRequestError: return swapped_provider if swapped_provider == original_provider: @@ -1108,6 +1119,7 @@ def _apply_prompt_management_to_responses_call( use_chat_completions_api, kwargs.get("model_info"), _api_base_kwarg(kwargs), + GenericLiteLLMParams(**kwargs), ), ): _stamp_injection_points_for_the_hook(kwargs, model, custom_llm_provider, input, tools) @@ -1142,7 +1154,7 @@ def _apply_prompt_management_to_responses_call( original_model=original_model, swapped_model=model, custom_llm_provider=custom_llm_provider, - kwargs=kwargs, + kwargs={**kwargs, "extra_headers": local_vars.get("extra_headers")}, prompt_id=prompt_id, ) ) @@ -1498,7 +1510,8 @@ def responses( model=model, api_base=local_vars.get("base_url", None), litellm_params=GenericLiteLLMParams( - **cast("dict[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict + extra_headers=extra_headers, + **cast("dict[str, object]", kwargs), # cast-ok: kwargs is the untyped request dict ), ) local_vars["custom_llm_provider"] = custom_llm_provider @@ -1514,8 +1527,16 @@ def responses( use_chat_completions_api=use_chat_completions_api, ) - # get llm provider logic - litellm_params: Final = GenericLiteLLMParams(**kwargs) + from litellm.litellm_core_utils.core_helpers import attribute_or_none + + litellm_params: Final = GenericLiteLLMParams(extra_headers=extra_headers, **kwargs) + provider_headers: Final = ResponsesAPIRequestUtils.merge_provider_specific_headers( + extra_headers=extra_headers, + provider_specific_header=attribute_or_none(litellm_params, "provider_specific_header"), + custom_llm_provider=custom_llm_provider, + ) + local_vars["extra_headers"] = provider_headers + affinity_api_base: Final = ( get_openai_credentials(api_base=litellm_params.api_base).api_base if custom_llm_provider == "openai" @@ -1525,7 +1546,7 @@ def responses( try: effective_extra_headers: Final = ( add_provider_affinity_header( - headers=extra_headers or MappingProxyType({}), + headers=provider_headers or MappingProxyType({}), litellm_params=MappingProxyType( { "provider_affinity_header": litellm_params.provider_affinity_header, @@ -1538,7 +1559,7 @@ def responses( default_header=default_affinity_header, ) if litellm_params.provider_affinity_header is not None or default_affinity_header is not None - else extra_headers + else provider_headers ) except ValueError as affinity_error: raise litellm.BadRequestError( diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index dc75e77d685..2d0873a218a 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -12,6 +12,7 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path +from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.types.llms.openai import ( AllMessageValues, @@ -29,11 +30,16 @@ from litellm.types.utils import ( CacheCreationTokenDetails, CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, + ProviderSpecificHeader, SpecialEnums, Usage, text_tokens_without_nested_reasoning, ) +_PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[ + TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None] +] = TypeAdapter(ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None) + class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object): pass @@ -233,6 +239,20 @@ class ResponsesAPIRequestUtils: **extra_headers, } + @staticmethod + def merge_provider_specific_headers( + extra_headers: Mapping[str, object] | None, + provider_specific_header: object, + custom_llm_provider: str | None, + ) -> dict[str, object] | None: # mutable-ok: legacy Responses request fields require a dict + scoped_headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=_PROVIDER_SPECIFIC_HEADER_ADAPTER.validate_python(provider_specific_header), + custom_llm_provider=custom_llm_provider, + ) + if extra_headers is None and not scoped_headers: + return None + return {**(extra_headers or {}), **scoped_headers} + @staticmethod def _check_valid_arg( supported_params: list[str] | None, diff --git a/litellm/router.py b/litellm/router.py index 7993b5b921c..8d95e3e91d4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -98,7 +98,9 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, r from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_llm_provider_logic import ( declared_authenticating_provider, + inferred_provider, is_registered_custom_provider, + resolve_model_and_provider_for_metadata, ) from litellm.litellm_core_utils.hidden_params import ( HIDDEN_PARAMS_ATTR as _HIDDEN_PARAMS_ATTR, @@ -561,6 +563,7 @@ _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset( {"model", "api_base", "api_key", "api_version", "responses_api_version"} ) _ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" +_FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS: Final = frozenset({"chatgpt", "anthropic"}) _CLAUDE_CODE_SESSION_ID_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") _CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600 CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET: Final = "claude_code_session_router_binding" @@ -1440,6 +1443,9 @@ class Router: # Maps (team_id, team_public_model_name) -> list of indices in model_list self.team_model_to_deployment_indices: dict[tuple[str, str], list[int]] = {} self.team_public_model_names: frozenset[str] = frozenset() + # Guards model_list and the three index maps above: writers change them, and readers turn an + # index into a deployment, under it, so a lookup never lands between a removal and its re-index + self._deployment_index_lock = threading.RLock() # Initialize cache attributes that ``_invalidate_model_group_info_cache`` # and ``_invalidate_access_groups_cache`` touch *before* the first @@ -1477,15 +1483,16 @@ class Router: if model_list is not None: # set_model_list will build indices automatically self.set_model_list(model_list) - # Track this router so a price data reload can rebuild its deployments' - # cost-map entries from the list it is serving at that moment. - _live_routers.add(self) self.healthy_deployments: list = self.model_list for m in model_list: if "model" in m["litellm_params"]: self.deployment_latency_map[m["litellm_params"]["model"]] = 0 else: self.model_list: list = [] # initialize an empty list - to allow _add_deployment and delete_deployment to work + # Track this router so a price data reload can rebuild its deployments' + # cost-map entries from the list it is serving at that moment, whether + # those deployments came from model_list or were added later from the DB. + _live_routers.add(self) if allowed_fails is not None: self.allowed_fails = allowed_fails @@ -2180,8 +2187,8 @@ class Router: public_model_name: Final = deployment.model_info.team_public_model_name if team_id is None or public_model_name is None: return False - sibling_indices: Final = self.team_model_to_deployment_indices.get((team_id, public_model_name)) or () - routable_siblings: Final = self._filter_blocked_deployments([self.model_list[idx] for idx in sibling_indices]) + siblings: Final = self._team_deployments_named(team_id, public_model_name) or () + routable_siblings: Final = self._filter_blocked_deployments([dict(sibling) for sibling in siblings]) return len(routable_siblings) > 1 _OVERRIDABLE_ROUTING_STRATEGIES: frozenset[str] = frozenset({"simple-shuffle", *_DEFAULT_SELECTOR_ATTR_BY_STRATEGY}) @@ -4798,9 +4805,39 @@ class Router: deployment_litellm_model_name = deployment["litellm_params"]["model"] deployment_api_base = deployment["litellm_params"].get("api_base") deployment_model_name: Final = deployment["model_name"] - if is_clientside_credential(request_kwargs=kwargs): + from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils + + request_context: Final = cast( # cast-ok: the router's legacy request dict has string keys + Mapping[str, object], kwargs + ) + raw_deployment_params: Final[object] = deployment["litellm_params"] + deployment_context: Final = _LITELLM_PARAMS_ADAPTER.validate_python( + raw_deployment_params.model_dump(exclude_none=True) + if isinstance(raw_deployment_params, LiteLLM_Params) + else raw_deployment_params + ) + declared_provider: Final = deployment_context.get("custom_llm_provider") + declared_model: Final = deployment_context.get("model") + deployment_provider: Final = ( + declared_provider + if isinstance(declared_provider, str) + else inferred_provider(declared_model if isinstance(declared_model, str) else None) + ) + forwarded_authorization: Final = ( + ProviderSpecificHeaderUtils.forwarded_authorization( + request_context.get("provider_specific_header"), deployment_provider + ) + if deployment_provider in _FORWARDED_CREDENTIAL_ISOLATION_PROVIDERS + else None + ) + if is_clientside_credential(request_kwargs=kwargs) or forwarded_authorization is not None: deployment_pydantic_obj: Final = self._handle_clientside_credential( - deployment=deployment, kwargs=kwargs, function_name=function_name + deployment=deployment, + kwargs={ + **kwargs, + **({"api_key": forwarded_authorization} if forwarded_authorization is not None else {}), + }, + function_name=function_name, ) model_info = deployment_pydantic_obj.model_info.model_dump() deployment_litellm_model_name = deployment_pydantic_obj.litellm_params.model @@ -10827,20 +10864,17 @@ class Router: # O(k) via the name→indices map: only touch deployments whose name # is listed in `available_models`, instead of scanning model_list. for name in config.available_models: - indices = self.model_name_to_deployment_indices.get(name, []) - if not indices: + deployments = self.deployments_named(name) + if not deployments: continue - d = (self.model_list or [])[indices[0]] - mi = d.get("model_info") if isinstance(d, dict) else d.model_info - mi_dict: dict[str, Any] = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {}) + d = deployments[0] + mi_dict: dict[str, Any] = d.get("model_info") or {} prefs_raw = mi_dict.get("adaptive_router_preferences") if prefs_raw is not None: model_to_prefs[name] = AdaptiveRouterPreferences(**prefs_raw) # model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set. - lp = d.get("litellm_params") if isinstance(d, dict) else d.litellm_params - lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {}) - cost = lp_dict.get("input_cost_per_token") + cost = d.get("litellm_params", {}).get("input_cost_per_token") if cost is None: cost = mi_dict.get("input_cost_per_token") if cost is not None: @@ -10932,10 +10966,11 @@ class Router: def set_model_list(self, model_list: list): original_model_list: Final = copy.deepcopy(model_list) self._discovered_model_info_cache.flush_cache() - self.model_list = [] - self.model_id_to_deployment_index_map = {} # Reset the index - self.model_name_to_deployment_indices = {} # Reset the model_name index - self.team_model_to_deployment_indices = {} # Reset the team_model index + with self._deployment_index_lock: + self.model_list = [] + self.model_id_to_deployment_index_map = {} # Reset the index + self.model_name_to_deployment_indices = {} # Reset the model_name index + self.team_model_to_deployment_indices = {} # Reset the team_model index self.team_pattern_routers = {} self.team_public_model_names = frozenset() # Reset per-strategy router registries so hot-reload doesn't leave @@ -11047,6 +11082,10 @@ class Router: custom_llm_provider = None dynamic_api_key = None api_base = None + elif declared_authenticating_provider(litellm_model, deployment.litellm_params.custom_llm_provider) is not None: + _model, custom_llm_provider = ( # rebind-ok: each branch resolves the deployment provider + self._model_group_llm_provider(deployment.litellm_params) + ) else: # check if model provider in supported providers ( @@ -11326,26 +11365,27 @@ class Router: - model: dict - the model to add to the list - model_id: Optional[str] - the model ID to use for indexing. If None, will try to get from model["model_info"]["id"] """ - idx: Final = len(self.model_list) - self.model_list.append(model) - self._invalidate_model_group_info_cache() - self._invalidate_access_groups_cache() + with self._deployment_index_lock: + idx: Final = len(self.model_list) + self.model_list.append(model) + self._invalidate_model_group_info_cache() + self._invalidate_access_groups_cache() - # Update model_id index for O(1) lookup - if model_id is not None: - self.model_id_to_deployment_index_map[model_id] = idx - elif model.get("model_info", {}).get("id") is not None: - self.model_id_to_deployment_index_map[model["model_info"]["id"]] = idx + # Update model_id index for O(1) lookup + if model_id is not None: + self.model_id_to_deployment_index_map[model_id] = idx + elif model.get("model_info", {}).get("id") is not None: + self.model_id_to_deployment_index_map[model["model_info"]["id"]] = idx - # Update model_name index for O(1) lookup - model_name: Final = model.get("model_name") - if model_name: - if model_name not in self.model_name_to_deployment_indices: - self.model_name_to_deployment_indices[model_name] = [] - self.model_name_to_deployment_indices[model_name].append(idx) + # Update model_name index for O(1) lookup + model_name: Final = model.get("model_name") + if model_name: + if model_name not in self.model_name_to_deployment_indices: + self.model_name_to_deployment_indices[model_name] = [] + self.model_name_to_deployment_indices[model_name].append(idx) - # Update team_model index for O(1) team-scoped lookup - self._update_team_model_index(model, idx) + # Update team_model index for O(1) team-scoped lookup + self._update_team_model_index(model, idx) def upsert_deployment(self, deployment: Deployment) -> Deployment | None: """ @@ -11372,18 +11412,15 @@ class Router: # if there is a new litellm param -> then update the deployment # remove the previous deployment - removal_idx: int | None = None - deployment_id: Final = deployment.model_info.id - deployment_fast_mapping: Final = self.model_id_to_deployment_index_map - - if deployment_id in deployment_fast_mapping: - removal_idx = deployment_fast_mapping[deployment_id] - + with self._deployment_index_lock: + removal_idx: Final = self.model_id_to_deployment_index_map.get(_deployment_model_id) if removal_idx is not None: self.model_list.pop(removal_idx) self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() - self._update_deployment_indices_after_removal(model_id=deployment_id, removal_idx=removal_idx) + self._update_deployment_indices_after_removal( + model_id=_deployment_model_id, removal_idx=removal_idx + ) # Free the outgoing deployment's pre-routing strategy slot (keyed by the # OLD model_name/tags) before the re-add below re-registers it. @@ -11617,35 +11654,32 @@ class Router: - The deleted deployment - OR None (if deleted deployment not found) """ - deployment_idx = None - if id in self.model_id_to_deployment_index_map: - deployment_idx = self.model_id_to_deployment_index_map[id] - try: - if deployment_idx is not None: + with self._deployment_index_lock: + deployment_idx: Final = self.model_id_to_deployment_index_map.get(id) + if deployment_idx is None: + return None # Pop the item from the list first item: Final = self.model_list.pop(deployment_idx) self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() self._update_deployment_indices_after_removal(model_id=id, removal_idx=deployment_idx) - _budget_limiter: Final = self.get_router_deployment_budget_limiter() - if _budget_limiter is not None: - _budget_limiter.unregister_deployment_budget(model_id=id) - try: - self._unregister_pre_routing_strategy_for_deployment( - deployment=item if isinstance(item, Deployment) else Deployment(**item) - ) - except Exception: - verbose_router_logger.exception( - "delete_deployment: could not release pre-routing strategies for model_id=%s; " - "the deployment is out of the model_list and its indices are repaired", - id, - ) - return item - else: - return None + _budget_limiter: Final = self.get_router_deployment_budget_limiter() + if _budget_limiter is not None: + _budget_limiter.unregister_deployment_budget(model_id=id) + try: + self._unregister_pre_routing_strategy_for_deployment( + deployment=item if isinstance(item, Deployment) else Deployment(**item) + ) + except Exception: + verbose_router_logger.exception( + "delete_deployment: could not release pre-routing strategies for model_id=%s; " + "the deployment is out of the model_list and its indices are repaired", + id, + ) except Exception: return None + return item def get_router_deployment_budget_limiter( self, @@ -11694,6 +11728,18 @@ class Router: if _budget_limiter is not None: _budget_limiter.register_deployment_budget(deployment=deployment.to_json(exclude_none=True)) + def deployments_named(self, model_name: str) -> tuple[DeploymentTypedDict, ...] | None: + """The deployments indexed under `model_name`, read under the index lock, or None when the name is not indexed.""" + with self._deployment_index_lock: + indices: Final = self.model_name_to_deployment_indices.get(model_name) + return None if indices is None else tuple(self.model_list[idx] for idx in indices) + + def _team_deployments_named(self, team_id: str, public_model_name: str) -> tuple[DeploymentTypedDict, ...] | None: + """The deployments `team_id` serves under `public_model_name`, or None when that pair is not indexed.""" + with self._deployment_index_lock: + indices: Final = self.team_model_to_deployment_indices.get((team_id, public_model_name)) + return None if indices is None else tuple(self.model_list[idx] for idx in indices) + def get_deployment(self, model_id: str) -> Deployment | None: """ Returns -> Deployment or None @@ -11701,9 +11747,10 @@ class Router: Raise Exception -> if model found in invalid format """ # Use O(1) lookup via model_id_to_deployment_index_map only - if model_id in self.model_id_to_deployment_index_map: - idx: Final = self.model_id_to_deployment_index_map[model_id] - model: Final = self.model_list[idx] + with self._deployment_index_lock: + idx: Final = self.model_id_to_deployment_index_map.get(model_id) + model: Final = None if idx is None else self.model_list[idx] + if model is not None: if isinstance(model, dict): return Deployment(**model) elif isinstance(model, Deployment): @@ -11737,17 +11784,17 @@ class Router: Optimized with O(1) index lookup instead of O(n) linear scan. """ # O(1) lookup in model_name index - if model_group_name in self.model_name_to_deployment_indices: - indices: Final = self.model_name_to_deployment_indices[model_group_name] - if indices: - # Return first deployment for this model_name - model: Final = self.model_list[indices[0]] - if isinstance(model, dict): - return Deployment(**model) - elif isinstance(model, Deployment): - return model - else: - raise Exception(f"Model Name invalid - {type(model)}") + with self._deployment_index_lock: + indices: Final = self.model_name_to_deployment_indices.get(model_group_name) + # Return first deployment for this model_name + model: Final = self.model_list[indices[0]] if indices else None + if model is not None: + if isinstance(model, dict): + return Deployment(**model) + elif isinstance(model, Deployment): + return model + else: + raise Exception(f"Model Name invalid - {type(model)}") return None @staticmethod @@ -11770,11 +11817,12 @@ class Router: by other teams so a shared model name never resolves another team's credentials. """ - indices: Final = self.model_name_to_deployment_indices.get(model_group_name) or () - usable: Final = ( - self.model_list[idx] for idx in indices if self.deployment_usable_by_team(self.model_list[idx], team_id) - ) - first_usable: Final = next(usable, None) + with self._deployment_index_lock: + indices: Final = self.model_name_to_deployment_indices.get(model_group_name) or () + usable: Final = ( + self.model_list[idx] for idx in indices if self.deployment_usable_by_team(self.model_list[idx], team_id) + ) + first_usable: Final = next(usable, None) if first_usable is None: return None return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable @@ -11877,11 +11925,10 @@ class Router: this never triggers pattern matching or deep copies, so it is safe to call per listed model on the /v1/models hot path. """ - indices: Final = self.model_name_to_deployment_indices.get(model_name) - if not indices: + deployments: Final = self.deployments_named(model_name) + if not deployments: return None - deployments: Final = tuple(self.model_list[index] for index in indices) model_infos: Final = tuple( MappingProxyType( { @@ -12054,10 +12101,11 @@ class Router: def _get_team_public_name_deployment(self, model_id: str, team_id: str | None) -> Deployment | None: if team_id is None: return None - team_indices: Final = self.team_model_to_deployment_indices.get((team_id, model_id)) - if not team_indices: + with self._deployment_index_lock: + team_indices: Final = self.team_model_to_deployment_indices.get((team_id, model_id)) + team_model: Final = self.model_list[team_indices[0]] if team_indices else None + if team_model is None: return None - team_model: Final = self.model_list[team_indices[0]] return Deployment(**team_model) if isinstance(team_model, dict) else team_model def _get_wildcard_deployment_usable_by_team(self, model_id: str, team_id: str | None) -> Deployment | None: @@ -12217,7 +12265,6 @@ class Router: _fallback_entry = litellm.model_cost.get(_azure_fallback_key) _fallback_resolves = _fallback_entry is not None and ( (_fallback_entry.get("max_input_tokens") or 0) > 0 - or (_fallback_entry.get("max_tokens") or 0) > 0 or (_fallback_entry.get("input_cost_per_token") or 0) > 0 ) if _fallback_resolves: @@ -12296,10 +12343,9 @@ class Router: Optimized with O(1) index lookup instead of O(n) linear scan. """ # O(1) lookup via model_id_to_deployment_index_map - if id in self.model_id_to_deployment_index_map: - idx: Final = self.model_id_to_deployment_index_map[id] - return self.model_list[idx] - return None + with self._deployment_index_lock: + idx: Final = self.model_id_to_deployment_index_map.get(id) + return None if idx is None else self.model_list[idx] def get_model_group(self, id: str) -> list | None: """ @@ -12392,6 +12438,17 @@ class Router: ) return model_info + @staticmethod + def _model_group_llm_provider(litellm_params: LiteLLM_Params) -> tuple[str, str]: + try: + return resolve_model_and_provider_for_metadata( + model=litellm_params.model, + custom_llm_provider=litellm_params.custom_llm_provider, + ) + except litellm.exceptions.BadRequestError as e: + verbose_router_logger.error("litellm.router.py::get_model_group_info() - %s", e) + return "", "" + def _set_model_group_info( self, model_group: str, user_facing_model_group_name: str, team_id: str | None = None ) -> ModelGroupInfo | None: @@ -12480,15 +12537,7 @@ class Router: deployment_is_mapped = deployment_is_catalog_mapped(model_info, model_info_dict) # get llm provider - litellm_model, llm_provider = "", "" - try: - litellm_model, llm_provider, _, _ = litellm.get_llm_provider( - model=litellm_params.model, - custom_llm_provider=litellm_params.custom_llm_provider, - litellm_params=litellm_params, - ) - except litellm.exceptions.BadRequestError as e: - verbose_router_logger.error("litellm.router.py::get_model_group_info() - %s", e) + litellm_model, llm_provider = self._model_group_llm_provider(litellm_params) if model_info is None: supported_openai_params = litellm.get_supported_openai_params( @@ -12969,20 +13018,19 @@ class Router: if model_name is not None: # O(1) lookup in model_name index, then O(k) iteration where k = deployments for this model_name - if model_name in self.model_name_to_deployment_indices: - indices: Final = self.model_name_to_deployment_indices[model_name] - for idx in indices: - model = self.model_list[idx] - if "model_info" in model and "id" in model["model_info"]: - if exclude_team_models and model["model_info"].get("team_id"): - continue - ids.append(model["model_info"]["id"]) + for model in self.deployments_named(model_name) or (): + if "model_info" in model and "id" in model["model_info"]: + if exclude_team_models and model["model_info"].get("team_id"): + continue + ids.append(model["model_info"]["id"]) else: # When model_name is None, return all model IDs # Use the index map keys for O(n) where n = total deployments - for model_id in self.model_id_to_deployment_index_map: - idx = self.model_id_to_deployment_index_map[model_id] - model = self.model_list[idx] + with self._deployment_index_lock: + id_indices: Final = tuple(self.model_id_to_deployment_index_map.items()) + models: Final = tuple(self.model_list) + for model_id, idx in id_indices: + model = models[idx] if "model_info" in model and "id" in model["model_info"]: if exclude_team_models and model["model_info"].get("team_id"): continue @@ -13260,7 +13308,9 @@ class Router: # handled downstream by the pattern_router in _common_checks_available_deployment. return None - def should_include_deployment(self, model_name: str, model: dict, team_id: str | None = None) -> bool: + def should_include_deployment( + self, model_name: str, model: DeploymentTypedDict | dict, team_id: str | None = None + ) -> bool: """ Get the team-specific model name if team_id matches the deployment. """ @@ -13308,12 +13358,10 @@ class Router: # O(1) lookup in team_model index when team_id is provided if team_id is not None: - key: Final = (team_id, model_name) - if key in self.team_model_to_deployment_indices: - indices = self.team_model_to_deployment_indices[key] + team_models: Final = self._team_deployments_named(team_id, model_name) + if team_models is not None: # O(k) where k = team deployments for this model_name (typically 1-10) - for idx in indices: - model = self.model_list[idx] + for model in team_models: if not self.should_include_deployment(model_name=model_name, model=model, team_id=team_id): continue if model_alias is not None: @@ -13326,12 +13374,10 @@ class Router: return returned_models # O(1) lookup in model_name index - if model_name in self.model_name_to_deployment_indices: - indices = self.model_name_to_deployment_indices[model_name] - + named_models: Final = self.deployments_named(model_name) + if named_models is not None: # O(k) where k = deployments for this model_name (typically 1-10) - for idx in indices: - model = self.model_list[idx] + for model in named_models: if self.should_include_deployment(model_name=model_name, model=model, team_id=team_id): if model_alias is not None: # Optimized: Use shallow copy since we only modify top-level model_name @@ -13345,7 +13391,7 @@ class Router: # Fallback: if team_id is provided and model_name not in index, # check if model_name matches any team_public_model_name # O(n) scan but only when team_id lookup fails - for idx, model in enumerate(self.model_list): + for _idx, model in enumerate(self.model_list): if self.should_include_deployment(model_name=model_name, model=model, team_id=team_id): if model_alias is not None: # Optimized: Use shallow copy since we only modify top-level model_name @@ -13743,21 +13789,21 @@ class Router: return [{**DeploymentTypedDict(**m), "model_name": model_name} for m in matches] def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: - """The deployments routing serves ``model`` from through a wildcard: the most specific - proxy-wide pattern matching it, else ``team_id``'s own pattern, never both.""" - proxy_wide: Final[Sequence[Mapping[str, object]]] = self.pattern_router.get_deployments_by_pattern(model=model) + """The deployments routing serves ``model`` from through a wildcard: ``team_id``'s own pattern + matching it, else the most specific proxy-wide pattern, never both.""" team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None - if proxy_wide or team_router is None: - return proxy_wide - return team_router.get_deployments_by_pattern(model=model) + team_owned: Final[Sequence[Mapping[str, object]]] = ( + team_router.get_deployments_by_pattern(model=model) if team_router is not None else () + ) + return team_owned or self.pattern_router.get_deployments_by_pattern(model=model) def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: """The models wildcard_route_deployments would route ``model`` to, without copying the deployments.""" - proxy_wide: Final = self.pattern_router.get_models_by_pattern(model=model) team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None - if proxy_wide or team_router is None: - return proxy_wide - return team_router.get_models_by_pattern(model=model) + team_owned: Final = ( + team_router.get_models_by_pattern(model=model) if team_router is not None else frozenset[str]() + ) + return team_owned or self.pattern_router.get_models_by_pattern(model=model) def team_ptu_ceiling(self, team_id: str, model_group: str) -> PTUTeamCeiling | None: """The per-minute ceiling ``team_id``'s PTU shares set on the group serving ``model_group``, @@ -14610,12 +14656,13 @@ class Router: def _team_deployments_across_teams(self, model: str) -> list[DeploymentTypedDict]: """Every team's deployments under public name `model`, for a proxy admin calling without a team.""" - team_deployments: Final = [ - self.model_list[index] - for (_, public_model_name), indices in self.team_model_to_deployment_indices.items() - if public_model_name == model - for index in indices - ] + with self._deployment_index_lock: + team_deployments: Final = [ + self.model_list[index] + for (_, public_model_name), indices in self.team_model_to_deployment_indices.items() + if public_model_name == model + for index in indices + ] team_ids: Final = { team_id for deployment in team_deployments diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index a475358e1f0..cc592bbcbc5 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -3126,17 +3126,14 @@ class ComplexityRouter(CustomLogger): model_to_prefs: Final[dict[str, AdaptiveRouterPreferences]] = {} model_to_cost: Final[dict[str, float]] = {} - model_list: Final = getattr(self.litellm_router_instance, "model_list", None) or [] - name_to_indices: Final = getattr(self.litellm_router_instance, "model_name_to_deployment_indices", {}) or {} for name in available_models: - indices = name_to_indices.get(name, []) - if not indices: + deployments = self.litellm_router_instance.deployments_named(name) + if not deployments: model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[]) model_to_cost[name] = 0.0 continue - deployment = model_list[indices[0]] - mi = deployment.get("model_info") if isinstance(deployment, dict) else deployment.model_info - mi_dict: dict[str, Any] = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {}) + deployment = deployments[0] + mi_dict: dict[str, Any] = deployment.get("model_info") or {} prefs_raw = mi_dict.get("adaptive_router_preferences") if prefs_raw is not None: model_to_prefs[name] = AdaptiveRouterPreferences(**prefs_raw) @@ -3144,9 +3141,7 @@ class ComplexityRouter(CustomLogger): model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[]) # model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set. - lp = deployment.get("litellm_params") if isinstance(deployment, dict) else deployment.litellm_params - lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {}) - cost = lp_dict.get("input_cost_per_token") + cost = deployment.get("litellm_params", {}).get("input_cost_per_token") if cost is None: cost = mi_dict.get("input_cost_per_token") model_to_cost[name] = float(cost) if cost is not None else 0.0 diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 4a2c56daf00..ec960a19f79 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -7,12 +7,12 @@ All values are configurable via proxy config.yaml. import math import re -import warnings +import sys from collections.abc import Iterable, Mapping from dataclasses import dataclass from enum import Enum from types import MappingProxyType -from typing import Annotated, Final, Literal, NamedTuple +from typing import TYPE_CHECKING, Annotated, Final, Literal, NamedTuple from pydantic import ( ConfigDict, @@ -27,8 +27,13 @@ from pydantic import ( from litellm.types.llms.base import LiteLLMBaseModel -with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) +if TYPE_CHECKING: + import sre_constants + import sre_parse +elif sys.version_info >= (3, 11): + from re import _constants as sre_constants + from re import _parser as sre_parse +else: import sre_constants import sre_parse diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index b5b80ddae6d..add0f78fd3a 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -14,7 +14,7 @@ bare string with no provider beside them; an operator who writes ``deepseek-r1`` Azure would otherwise be priced against whoever else owns that name. """ -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Final, NamedTuple from litellm._logging import verbose_router_logger @@ -80,12 +80,11 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]: straight at a provider model rather than at a configured group, and prices under its own name because there is no deployment to override it. """ - indices: Final = router.model_name_to_deployment_indices.get(group_name) - if not indices: + deployments: Final = router.deployments_named(group_name) + if not deployments: return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else () - def candidate(index: int) -> Baseline | None: - deployment: Final = router.model_list[index] + def candidate(deployment: Mapping[str, object]) -> Baseline | None: params: Final = deployment.get("litellm_params") if not isinstance(params, dict): return None @@ -98,7 +97,7 @@ def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]: deployment_id: Final = info.get("id") if isinstance(info, dict) else None return Baseline(qualified, str(deployment_id) if deployment_id else None) - return tuple(c for index in indices if (c := candidate(index)) is not None) + return tuple(c for deployment in deployments if (c := candidate(deployment)) is not None) def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | None: diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 086f5517c29..2e350a2f87b 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -148,7 +148,7 @@ def _model_max_output_tokens(model_name: str) -> int | None: # this lookup is a fallback default and must never fail the request. with contextlib.suppress(Exception): info: Final = litellm.get_model_info(model=model_name) - model_max: Final = info.get("max_output_tokens") or info.get("max_tokens") + model_max: Final = info.get("max_output_tokens") if model_max is not None: return max(0, int(model_max)) return None diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 4357722970f..16b8a22a40f 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -469,7 +469,7 @@ def _supports_none_reasoning_effort(model_info: Mapping[str, object], flag: obje if model_info.get("litellm_provider") != "azure": return flag is not False - from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config + from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config key: Final = model_info.get("key") if not isinstance(key, str) or not AzureOpenAIGPT5Config.is_model_gpt_5_model(key): diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 089f21a33c1..717b74e5741 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -91,6 +91,7 @@ class ProviderConnection: litellm_credential_name: str | None = None configurable_clientside_auth_params: "Sequence[str | ConfigurableClientsideParamsCustomAuth] | None" = None use_xai_oauth: bool | None = None + chatgpt_auth_file: str | None = None github_copilot_auth_type: str | None = None github_copilot_host: str | None = None github_copilot_user_session: object | None = None diff --git a/litellm/types/proxy/management_endpoints/key_requests.py b/litellm/types/proxy/management_endpoints/key_requests.py index baa02b68d98..172357d8ac4 100644 --- a/litellm/types/proxy/management_endpoints/key_requests.py +++ b/litellm/types/proxy/management_endpoints/key_requests.py @@ -55,6 +55,8 @@ class LiteLLMKeyType(str, enum.Enum): class GenerateKeyRequest(KeyRequestBase): + tpm_limit: int | None = Field(default=None, ge=0) + rpm_limit: int | None = Field(default=None, ge=0) soft_budget: float | None = None send_invite_email: bool | None = None key_type: LiteLLMKeyType | None = Field( @@ -82,6 +84,8 @@ def _is_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: class UpdateKeyRequest(KeyRequestBase): + tpm_limit: int | None = Field(default=None, ge=0) + rpm_limit: int | None = Field(default=None, ge=0) # Note: the defaults of all Params here MUST BE NONE # else they will get overwritten duration: str | None = None diff --git a/litellm/types/proxy/management_endpoints/model_management_endpoints.py b/litellm/types/proxy/management_endpoints/model_management_endpoints.py index c012f872280..f032bb20c2a 100644 --- a/litellm/types/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/model_management_endpoints.py @@ -1,11 +1,37 @@ from collections.abc import Mapping from datetime import datetime +from typing import Annotated, Final -from pydantic import ConfigDict, Field +from pydantic import ConfigDict, Field, model_validator from litellm.types.llms.base import LiteLLMBaseModel -from ...router import ModelGroupInfo +from ...router import ModelGroupInfo, updateDeployment + + +class PatchModelRequest(updateDeployment): + litellm_params_to_remove: tuple[Annotated[str, Field(min_length=1)], ...] = Field( + default=(), + description=( + "Top-level litellm_params keys to delete. Omitted parameters remain unchanged. " + "The required model key and keys also supplied in litellm_params cannot be removed. " + "Removing an absent key or repeating a key is a no-op." + ), + ) + + model_config = ConfigDict(frozen=True, protected_namespaces=()) + + @model_validator(mode="after") + def validate_param_removals(self) -> "PatchModelRequest": + removals: Final = frozenset(self.litellm_params_to_remove) + if "model" in removals: + raise ValueError("The required litellm_params.model cannot be removed") + conflicts: Final[frozenset[str]] = ( + removals.intersection(self.litellm_params.model_fields_set) if self.litellm_params else frozenset() + ) + if conflicts: + raise ValueError(f"Cannot set and remove the same litellm_params keys: {', '.join(sorted(conflicts))}") + return self class ModelGroupInfoProxy(ModelGroupInfo): diff --git a/litellm/types/proxy/management_endpoints/user_requests.py b/litellm/types/proxy/management_endpoints/user_requests.py index 49dda5995bc..e53dc4152c3 100644 --- a/litellm/types/proxy/management_endpoints/user_requests.py +++ b/litellm/types/proxy/management_endpoints/user_requests.py @@ -16,6 +16,8 @@ class NewUserRequestTeam(LiteLLMPydanticObjectBase): class NewUserRequest(GenerateRequestBase): + tpm_limit: int | None = Field(default=None, ge=0) + rpm_limit: int | None = Field(default=None, ge=0) max_budget: float | None = None user_email: str | None = None user_alias: str | None = None @@ -47,6 +49,8 @@ class NewUserRequest(GenerateRequestBase): class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase): # shared with BulkUpdateUserRequest + tpm_limit: int | None = Field(default=None, ge=0) + rpm_limit: int | None = Field(default=None, ge=0) # repr=False keeps the plaintext out of management-endpoint alerts, which str() the request model password: str | None = Field(default=None, repr=False) spend: float | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index cd1389aea9d..ba9b82e2a57 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -572,6 +572,10 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): default=False, description="Use stored xAI OAuth credentials when no xAI API key is configured.", ) + chatgpt_auth_file: str | None = Field( + default=None, + description="Path to a ChatGPT OAuth auth.json for this deployment.", + ) fireworks_forward_user_id: bool | None = Field( default=None, description="Send the LiteLLM user id of the calling key as the `user` field on Fireworks AI chat, responses and messages requests.", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cc7ef851111..7e8645fd0dd 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4360,6 +4360,7 @@ server_owned_wif_litellm_params: Final = ( + openai_wif_litellm_params + oauth_token_exchange_litellm_params + github_copilot_oauth_litellm_params + + ("chatgpt_auth_file",) ) secret_bearing_wif_litellm_params: Final = tuple(sorted(WIF_SECRET_BEARING_KEYS)) @@ -4414,6 +4415,7 @@ GenericBudgetConfigType = dict[str, BudgetConfig] class LlmProviders(str, Enum): + REFUSELESS = "refuseless" OPENAI = "openai" CHATGPT = "chatgpt" OPENAI_LIKE = "openai_like" # embedding only @@ -4441,6 +4443,7 @@ class LlmProviders(str, Enum): VERTEX_AI = "vertex_ai" VERTEX_AI_BETA = "vertex_ai_beta" GEMINI = "gemini" + ABLITERATION = "abliteration" AI21 = "ai21" BASETEN = "baseten" BLACK_FOREST_LABS = "black_forest_labs" diff --git a/litellm/utils.py b/litellm/utils.py index e2820f9c682..1aed15ba83e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2964,12 +2964,14 @@ def supports_native_streaming(model: str, custom_llm_provider: str | None) -> bo Raises: Exception: If the given model is not found in model_prices_and_context_window.json. """ + from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata + try: - model, custom_llm_provider, _, _ = litellm.get_llm_provider( + resolved_model, resolved_provider = resolve_model_and_provider_for_metadata( model=model, custom_llm_provider=custom_llm_provider ) - model_info: Final = get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + model_info: Final = get_model_info_helper(model=resolved_model, custom_llm_provider=resolved_provider) supports_native_streaming = model_info.get("supports_native_streaming", True) if supports_native_streaming is None: supports_native_streaming = True @@ -6077,11 +6079,8 @@ def get_max_tokens(model: str) -> int | None: return None try: - if model in litellm.model_cost: - if "max_output_tokens" in litellm.model_cost[model]: - return litellm.model_cost[model]["max_output_tokens"] - elif "max_tokens" in litellm.model_cost[model]: - return litellm.model_cost[model]["max_tokens"] + if model in litellm.model_cost and "max_output_tokens" in litellm.model_cost[model]: + return litellm.model_cost[model]["max_output_tokens"] get_llm_provider: Final = litellm_utils.get_llm_provider model, custom_llm_provider, _, _ = get_llm_provider(model=model) if custom_llm_provider == "huggingface": @@ -6090,8 +6089,6 @@ def get_max_tokens(model: str) -> int | None: if model in litellm.model_cost: # check if extracted model is in model_list if "max_output_tokens" in litellm.model_cost[model]: return litellm.model_cost[model]["max_output_tokens"] - elif "max_tokens" in litellm.model_cost[model]: - return litellm.model_cost[model]["max_tokens"] else: raise Exception() return None @@ -6740,7 +6737,7 @@ def get_model_info_helper( returned_model_info: Final = ModelInfoBase( key=key, - max_tokens=_model_info.get("max_tokens", None), + max_tokens=_model_info.get("max_output_tokens") or _model_info.get("max_input_tokens"), max_input_tokens=_model_info.get("max_input_tokens", None), max_output_tokens=_model_info.get("max_output_tokens", None), input_cost_per_token=_input_cost_per_token, @@ -8240,9 +8237,7 @@ def trim_messages( if max_tokens is None: # Check if model is valid if model in litellm.model_cost: - max_tokens_for_model: Final = litellm.model_cost[model].get( - "max_input_tokens", litellm.model_cost[model]["max_tokens"] - ) + max_tokens_for_model: Final = litellm.model_cost[model]["max_input_tokens"] max_tokens = int(max_tokens_for_model * trim_ratio) else: # if user did not specify max (input) tokens @@ -9571,7 +9566,7 @@ class ProviderConfigManager: return get_nvidia_nim_rerank_config(model) elif litellm.LlmProviders.VERTEX_AI == provider: - return litellm.VertexAIRerankConfig() + return ProviderConfigManager._get_vertex_ai_rerank_config_cached() elif litellm.LlmProviders.FIREWORKS_AI == provider: return litellm.FireworksAIRerankConfig() elif litellm.LlmProviders.VOYAGE == provider: @@ -9590,6 +9585,11 @@ class ProviderConfigManager: return get_dashscope_family_rerank_config(provider.value) return litellm.CohereRerankConfig() + @staticmethod + @lru_cache(maxsize=1) + def _get_vertex_ai_rerank_config_cached() -> BaseRerankConfig: + return litellm.VertexAIRerankConfig() + @staticmethod def get_provider_decisions_config(model: str, provider: LlmProviders) -> BaseDecisionsConfig | None: if provider == LlmProviders.PERPLEXITY: diff --git a/migrations/Dockerfile b/migrations/Dockerfile index 84ea67fcc5f..2b1fd029143 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -137,7 +137,9 @@ ENV HOME=/home/nonroot \ PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ PRISMA_OFFLINE_MODE=true -COPY --from=builder --chown=nonroot:nonroot /app /app +COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv +COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm +COPY --from=builder --chown=nonroot:nonroot /app/run.py /app/run.py COPY --from=builder /opt/prisma /opt/prisma RUN chmod -R a+rX /opt/prisma && \ diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9a7d88b8eda..b820b058eb3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9,9 +9,8 @@ "input_cost_per_audio_token": 0.0, "input_cost_per_token": 0.0, "litellm_provider": "one of https://docs.litellm.ai/docs/providers", - "max_input_tokens": "max input tokens, if the provider specifies it. if not default to max_tokens", - "max_output_tokens": "max output tokens, if the provider specifies it. if not default to max_tokens", - "max_tokens": "LEGACY parameter. set to max_output_tokens if provider specifies it. IF not set to max_input_tokens, if provider specifies it.", + "max_input_tokens": "max input tokens, if the provider specifies it", + "max_output_tokens": "max output tokens, if the provider specifies it", "mode": "one of: chat, embedding, completion, image_generation, audio_transcription, audio_speech, image_generation, moderation, rerank, search", "output_cost_per_reasoning_token": 0.0, "output_cost_per_token": 0.0, @@ -49,37 +48,132 @@ "1024-x-1024/50-steps/stability.stable-diffusion-xl-v1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.04 }, "1024-x-1024/max-steps/stability.stable-diffusion-xl-v1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.08 }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.018 }, "512-x-512/max-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.036 }, + "abliteration/abliterated-model": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "abliteration", + "max_input_tokens": 262144, + "max_output_tokens": 262134, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://abliteration.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ] + }, + "abliteration/abliterated-model-large": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "abliteration", + "max_input_tokens": 1000000, + "max_output_tokens": 999990, + "mode": "chat", + "output_cost_per_token": 5e-06, + "source": "https://abliteration.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ] + }, + "abliteration/abliterated-model-large-v2": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "abliteration", + "max_input_tokens": 1000000, + "max_output_tokens": 999990, + "mode": "chat", + "output_cost_per_token": 5e-06, + "source": "https://abliteration.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_none_reasoning_effort": false, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ] + }, + "abliteration/abliterated-model-large-flash": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "abliteration", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://abliteration.ai/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "reasoning_effort_levels": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ] + }, "ai21.j2-mid-v1": { "input_cost_per_token": 1.25e-05, "litellm_provider": "bedrock", "max_input_tokens": 8191, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.25e-05, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -89,7 +183,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8191, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.88e-05, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -100,7 +193,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-large.md" @@ -111,7 +203,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-ai21-labs-jamba-1-5-mini.md" @@ -121,7 +212,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 70000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -298,7 +388,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", @@ -311,7 +400,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", @@ -324,7 +412,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x4.md", @@ -337,7 +424,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-writer-palmyra-x5.md", @@ -352,7 +438,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -368,7 +453,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07, @@ -389,7 +473,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -414,7 +497,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -435,7 +517,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.311e-06, "supports_function_calling": true, @@ -455,7 +536,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://aws.amazon.com/nova/pricing/", @@ -474,7 +554,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.597e-06, "supports_function_calling": true, @@ -494,7 +573,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://aws.amazon.com/nova/pricing/", @@ -518,7 +596,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-06, "output_cost_per_token_batches": 1.375e-06, @@ -544,7 +621,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -562,7 +638,6 @@ "amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 8172, - "max_tokens": 8172, "mode": "embedding", "input_cost_per_token": 1.35e-07, "input_cost_per_image": 6e-05, @@ -583,7 +658,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.4e-07, "output_cost_per_token_batches": 7e-08, @@ -602,7 +676,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.25e-05, "output_cost_per_token_batches": 6.25e-06, @@ -626,7 +699,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, "output_cost_per_token_batches": 1.6e-06, @@ -646,7 +718,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2.75e-06, @@ -660,7 +731,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -669,7 +739,6 @@ "input_cost_per_token": 8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128, - "max_tokens": 128, "metadata": { "notes": "'supports_image_input' is a deprecated field. Use 'supports_embedding_image_input' instead." }, @@ -684,7 +753,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -693,7 +761,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -702,7 +769,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -740,7 +806,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -755,7 +820,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -770,7 +834,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -784,7 +847,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 500, - "max_tokens": 500, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 512, @@ -798,7 +860,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 500, - "max_tokens": 500, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 512, @@ -812,7 +873,6 @@ "input_cost_per_image": 0.0001, "litellm_provider": "bedrock", "max_input_tokens": 500, - "max_tokens": 500, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 512, @@ -860,7 +920,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -869,7 +928,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-07 }, @@ -878,7 +936,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -889,7 +946,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -909,7 +965,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", @@ -941,7 +996,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -964,7 +1018,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -982,7 +1035,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -1001,7 +1053,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -1021,7 +1072,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -1039,7 +1089,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -1054,7 +1103,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-06, "supports_tool_choice": true @@ -1067,7 +1115,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -1097,7 +1144,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -1125,7 +1171,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1164,7 +1209,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1204,7 +1248,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1244,7 +1287,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1284,7 +1326,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1324,7 +1365,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1364,7 +1404,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1405,7 +1444,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -1427,7 +1465,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1470,7 +1507,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1513,7 +1549,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1556,7 +1591,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -1596,7 +1630,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1639,7 +1672,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1678,7 +1710,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1721,7 +1752,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -1760,7 +1790,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1803,7 +1832,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1842,7 +1870,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1880,7 +1907,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "search_context_cost_per_query": { @@ -1922,7 +1948,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -1965,7 +1990,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "search_context_cost_per_query": { @@ -2010,7 +2034,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -2053,7 +2076,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "search_context_cost_per_query": { @@ -2098,7 +2120,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2141,7 +2162,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2186,7 +2206,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2229,7 +2248,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2269,7 +2287,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2312,7 +2329,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2357,7 +2373,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2395,7 +2410,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -2435,7 +2449,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -2480,7 +2493,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -2525,7 +2537,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2570,7 +2581,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2615,7 +2625,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2660,7 +2669,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2703,7 +2711,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -2745,7 +2752,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -2790,7 +2796,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -2835,7 +2840,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -2880,7 +2884,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -2925,7 +2928,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -2970,7 +2972,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -3011,7 +3012,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3050,7 +3050,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3089,7 +3088,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3128,7 +3126,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3167,7 +3164,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3206,7 +3202,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3246,7 +3241,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3286,7 +3280,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3319,7 +3312,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05 }, @@ -3328,7 +3320,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -3338,7 +3329,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-07 }, @@ -3347,7 +3337,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -3356,7 +3345,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/codellama-CodeLlama-70b-Instruct-hf" @@ -3366,7 +3354,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/google-gemma-7b-it" @@ -3376,7 +3363,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07 }, @@ -3385,7 +3371,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -3394,7 +3379,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07 }, @@ -3403,7 +3387,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/meta-llama-Meta-Llama-3-70B-Instruct" @@ -3413,7 +3396,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/meta-llama-Meta-Llama-3-8B-Instruct" @@ -3423,7 +3405,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mistral-7B-Instruct-v0.1", @@ -3434,7 +3415,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mixtral-8x22B-Instruct-v0.1", @@ -3445,7 +3425,6 @@ "litellm_provider": "anyscale", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.anyscale.com/preview/endpoints/text-generation/supported-models/mistralai-Mixtral-8x7B-Instruct-v0.1", @@ -3457,7 +3436,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.52e-07, "supports_function_calling": true, @@ -3473,7 +3451,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.48e-07, "supports_function_calling": true, @@ -3489,7 +3466,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.36e-06, "output_cost_per_token_flex": 1.68e-06, @@ -3508,7 +3484,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -3543,7 +3518,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -3595,7 +3569,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -3628,7 +3601,6 @@ "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -3640,7 +3612,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 6e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -3670,7 +3641,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -3685,7 +3655,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -3709,7 +3678,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, @@ -3734,7 +3702,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3765,7 +3732,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3797,7 +3763,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3830,7 +3795,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3868,7 +3832,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3899,7 +3862,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3938,7 +3900,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -3971,7 +3932,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -3996,7 +3956,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -4030,7 +3989,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": false, @@ -4052,7 +4010,6 @@ "litellm_provider": "azure", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -4087,7 +4044,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, @@ -4105,7 +4061,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -4154,7 +4109,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -4203,7 +4157,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -4252,7 +4205,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -4304,7 +4256,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -4345,7 +4296,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", @@ -4395,7 +4345,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 6e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", @@ -4439,7 +4388,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -4486,7 +4434,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -4533,7 +4480,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -4581,7 +4527,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -4629,7 +4574,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -4675,7 +4619,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -4717,7 +4660,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_priority": 9e-06, @@ -4759,7 +4701,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_priority": 9e-06, @@ -4801,7 +4742,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_priority": 2.5e-06, @@ -4843,7 +4783,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_priority": 2.5e-06, @@ -4883,7 +4822,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/ where deployment-name is your Azure deployment (e.g., azure-model-router)" @@ -4895,7 +4833,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/aoai/", "comment": "Catalog-name twin of azure_ai/model_router: the flat $0.14 per M input tokens is the router's own fee, the routed model is priced on top of it" @@ -4908,7 +4845,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -4929,7 +4865,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -4949,7 +4884,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -4968,7 +4902,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -4987,7 +4920,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5006,7 +4938,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2.2e-05, "output_cost_per_token": 2.64e-06, @@ -5026,7 +4957,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 0.00022, "output_cost_per_token": 2.2e-05, @@ -5046,7 +4976,6 @@ "litellm_provider": "azure", "max_input_tokens": 16000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 8.8e-05, "output_cost_per_token": 2.2e-05, @@ -5076,7 +5005,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -5116,7 +5044,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "output_cost_per_token_batches": 1.1e-06, @@ -5155,7 +5082,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "supported_endpoints": [ @@ -5196,7 +5122,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5228,7 +5153,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5261,7 +5185,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, @@ -5298,7 +5221,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -5317,7 +5239,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -5335,7 +5256,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -5353,7 +5273,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -5371,7 +5290,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -5389,7 +5307,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5409,7 +5326,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5426,7 +5342,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -5443,7 +5358,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5463,7 +5377,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -5484,7 +5397,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -5525,7 +5437,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5557,7 +5468,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -5587,7 +5497,6 @@ "litellm_provider": "azure", "max_input_tokens": 4097, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -5597,7 +5506,7 @@ "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", "max_input_tokens": 4097, - "max_tokens": 4097, + "max_output_tokens": 4097, "mode": "completion", "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -5607,7 +5516,6 @@ "litellm_provider": "azure", "max_input_tokens": 4097, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -5619,7 +5527,6 @@ "litellm_provider": "azure", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5631,7 +5538,6 @@ "litellm_provider": "azure", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5642,7 +5548,7 @@ "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", "max_input_tokens": 4097, - "max_tokens": 4097, + "max_output_tokens": 4097, "mode": "completion", "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -5651,7 +5557,7 @@ "input_cost_per_token": 1.65e-06, "litellm_provider": "azure_text", "max_input_tokens": 4097, - "max_tokens": 4097, + "max_output_tokens": 4097, "mode": "completion", "output_cost_per_token": 2.2e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -5662,7 +5568,6 @@ "litellm_provider": "azure", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -5674,7 +5579,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -5687,7 +5591,6 @@ "litellm_provider": "azure", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5699,7 +5602,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -5712,7 +5614,6 @@ "litellm_provider": "azure", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.00012, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5724,7 +5625,6 @@ "litellm_provider": "azure", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.00012, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/legacy-models", @@ -5735,7 +5635,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/", @@ -5748,7 +5647,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/", @@ -5762,7 +5660,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.3e-05, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/", @@ -5779,7 +5676,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -5817,7 +5713,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -5855,7 +5750,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, @@ -5893,7 +5787,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, @@ -5929,7 +5822,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -5963,7 +5855,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -5997,7 +5888,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 0.00015, "output_cost_per_token_batches": 7.5e-05, @@ -6017,7 +5907,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -6034,7 +5923,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -6053,7 +5941,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -6073,7 +5960,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -6092,7 +5978,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -6125,7 +6010,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -6157,7 +6041,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -6190,7 +6073,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -6206,6 +6088,8 @@ "text", "audio" ], + "supports_audio_input": true, + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6223,7 +6107,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6255,7 +6138,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6270,6 +6152,8 @@ "text", "audio" ], + "supports_audio_input": true, + "supports_audio_output": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -6286,7 +6170,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -6318,7 +6201,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -6336,7 +6218,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -6354,7 +6235,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 6e-07, @@ -6388,7 +6268,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6410,7 +6289,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6445,7 +6323,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6479,7 +6356,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6514,7 +6390,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -6548,7 +6423,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -6583,7 +6457,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6618,7 +6491,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6652,7 +6524,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -6716,7 +6587,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 0.0002, "output_cost_per_token": 2e-05, @@ -6735,7 +6605,6 @@ "litellm_provider": "azure", "max_input_tokens": 16000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 2e-05, @@ -6789,7 +6658,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "audio_transcription", "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ @@ -6813,7 +6681,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -6856,7 +6723,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -6891,7 +6757,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "output_cost_per_token_priority": 3.6e-06, @@ -6928,7 +6793,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -6967,7 +6831,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -7004,7 +6867,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7040,7 +6902,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -7079,7 +6940,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -7118,7 +6978,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -7154,7 +7013,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -7190,7 +7048,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -7224,7 +7081,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -7265,7 +7121,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7290,7 +7145,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7312,7 +7166,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7333,7 +7186,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7354,7 +7206,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7376,7 +7227,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7397,7 +7247,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7418,7 +7267,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7440,7 +7288,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7461,7 +7308,6 @@ "litellm_provider": "azure", "max_input_tokens": 111616, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.54e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7482,7 +7328,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.6e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7498,7 +7343,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.6e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7514,7 +7358,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7546,7 +7389,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7578,7 +7420,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7614,7 +7455,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_batches": 7e-06, @@ -7654,7 +7494,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_batches": 7e-06, @@ -7692,7 +7531,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -7726,7 +7564,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -7758,7 +7595,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -7797,7 +7633,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -7846,7 +7681,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -7902,7 +7736,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -7952,7 +7785,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -8002,7 +7834,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -8058,7 +7889,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -8108,7 +7938,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -8159,7 +7988,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -8208,7 +8036,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -8262,7 +8089,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -8323,7 +8149,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -8387,7 +8212,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -8451,7 +8275,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -8515,7 +8338,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -8579,7 +8401,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, @@ -8643,7 +8464,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, @@ -8695,7 +8515,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -8744,7 +8563,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -8793,7 +8611,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -8842,7 +8659,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -8897,7 +8713,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -8950,7 +8765,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -8997,7 +8811,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -9046,7 +8859,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -9095,7 +8907,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -9144,7 +8955,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -9193,7 +9003,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -9238,7 +9047,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "reasoning_effort_levels": [ @@ -9275,7 +9083,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "reasoning_effort_levels": [ @@ -9321,7 +9128,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9376,7 +9182,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9432,7 +9237,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, @@ -9488,7 +9292,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, @@ -9538,7 +9341,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "output_cost_per_token_above_272k_tokens": 8.25e-05, @@ -9587,7 +9389,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "output_cost_per_token_above_272k_tokens": 8.25e-07, @@ -9642,7 +9443,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -9685,7 +9485,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.3e-05, "reasoning_effort_levels": [ @@ -9731,7 +9530,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9786,7 +9584,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "output_cost_per_token_above_272k_tokens": 3.3e-05, @@ -9842,7 +9639,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-05, "output_cost_per_token_above_272k_tokens": 1.98e-05, @@ -9898,7 +9694,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.32e-06, "output_cost_per_token_above_272k_tokens": 1.98e-06, @@ -9957,7 +9752,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -10016,7 +9810,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, @@ -10071,7 +9864,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, @@ -10123,7 +9915,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -10183,7 +9974,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_batches": 1.5e-05, @@ -10242,7 +10032,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10295,7 +10084,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10349,7 +10137,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10402,7 +10189,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -10449,7 +10235,6 @@ "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -10493,7 +10278,6 @@ "litellm_provider": "azure", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -10535,7 +10319,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "search_context_cost_per_query": { @@ -10586,7 +10369,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "search_context_cost_per_query": { @@ -10638,7 +10420,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "search_context_cost_per_query": { @@ -10686,7 +10467,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "search_context_cost_per_query": { @@ -11062,7 +10842,7 @@ "input_cost_per_token": 8e-06, "litellm_provider": "azure", "max_input_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_function_calling": true @@ -11071,7 +10851,7 @@ "input_cost_per_token": 8e-06, "litellm_provider": "azure", "max_input_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_function_calling": true @@ -11083,7 +10863,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_function_calling": true, @@ -11101,7 +10880,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -11120,7 +10898,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11138,7 +10915,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11156,7 +10932,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "supported_endpoints": [ @@ -11187,7 +10962,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -11219,7 +10993,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, "search_context_cost_per_query": { @@ -11257,7 +11030,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_prompt_caching": true, @@ -11274,7 +11046,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11291,7 +11062,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -11322,7 +11092,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -11353,7 +11122,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supported_endpoints": [ @@ -11384,7 +11152,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -11429,7 +11196,6 @@ "input_cost_per_token": 1.43e-07, "litellm_provider": "azure", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -11439,7 +11205,6 @@ "input_cost_per_token": 2.2e-08, "litellm_provider": "azure", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -11449,7 +11214,6 @@ "input_cost_per_token": 1.1e-07, "litellm_provider": "azure", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" @@ -11500,7 +11264,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -11538,7 +11301,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.76e-06, "output_cost_per_token_batches": 8.8e-07, @@ -11574,7 +11336,6 @@ "litellm_provider": "azure", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, @@ -11608,7 +11369,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11629,7 +11389,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11649,7 +11408,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11668,7 +11426,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11687,7 +11444,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11706,7 +11462,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2.2e-05, "output_cost_per_token": 2.64e-06, @@ -11726,7 +11481,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 0.00022, "output_cost_per_token": 2.2e-05, @@ -11746,7 +11500,6 @@ "litellm_provider": "azure", "max_input_tokens": 16000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 8.8e-05, "output_cost_per_token": 2.2e-05, @@ -11776,7 +11529,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, @@ -11816,7 +11568,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "output_cost_per_token_batches": 1.1e-06, @@ -11854,7 +11605,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, @@ -11892,7 +11642,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "supported_endpoints": [ @@ -11933,7 +11682,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11965,7 +11713,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -11998,7 +11745,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -12017,7 +11763,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.6e-05, "output_cost_per_token_batches": 3.3e-05, @@ -12035,7 +11780,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12053,7 +11797,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -12086,7 +11829,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -12119,7 +11861,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12137,7 +11878,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12155,7 +11895,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12176,7 +11915,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -12200,7 +11938,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -12211,7 +11948,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "azure_ai", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -12223,7 +11959,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12241,7 +11976,6 @@ "azure_ai/FLUX-1.1-pro": { "litellm_provider": "azure_ai", "max_input_tokens": 5000, - "max_tokens": 5000, "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659", @@ -12252,7 +11986,6 @@ "azure_ai/FLUX.1-Kontext-pro": { "litellm_provider": "azure_ai", "max_input_tokens": 5000, - "max_tokens": 5000, "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/black-forest-labs/", @@ -12263,7 +11996,6 @@ "azure_ai/flux.2-pro": { "litellm_provider": "azure_ai", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "image_generation", "output_cost_per_image": 0.04, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -12275,7 +12007,6 @@ "input_cost_per_pixel": 5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "image_generation", "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/black-forest-labs/", "supported_endpoints": [ @@ -12297,7 +12028,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.85e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12312,7 +12042,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.828e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12328,7 +12057,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.52e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12344,7 +12072,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12359,7 +12086,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.84e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12374,7 +12100,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 7.26e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12389,7 +12114,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.46e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12411,7 +12135,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.3e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12434,7 +12157,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12457,7 +12179,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12480,7 +12201,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ @@ -12509,7 +12229,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12524,7 +12243,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12567,7 +12285,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.64e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12633,7 +12350,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12656,7 +12372,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.1e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -12668,7 +12383,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12681,7 +12395,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 10000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/", @@ -12695,7 +12408,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 8192, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12707,7 +12419,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.54e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12719,7 +12430,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12732,7 +12442,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12745,7 +12454,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12758,7 +12466,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12771,7 +12478,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12784,7 +12490,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12797,7 +12502,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12810,7 +12514,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12823,7 +12526,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5.2e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12836,7 +12538,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12850,7 +12551,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12863,7 +12563,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3.2e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12877,7 +12576,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12889,7 +12587,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12931,7 +12628,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 8182, - "max_tokens": 8182, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -12971,7 +12667,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -12985,7 +12680,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" @@ -12997,7 +12691,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" @@ -13008,7 +12701,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076" @@ -13019,7 +12711,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076" @@ -13029,7 +12720,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13044,7 +12734,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13060,7 +12749,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.56e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13072,7 +12760,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13088,7 +12775,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 5.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13105,7 +12791,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13118,7 +12803,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 3072, @@ -13138,7 +12822,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13155,7 +12838,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -13176,7 +12858,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -13196,7 +12877,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13215,7 +12895,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13232,7 +12911,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, @@ -13246,7 +12924,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, @@ -13260,7 +12937,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -13275,7 +12951,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.00971, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models" @@ -13286,7 +12961,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 70000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13298,7 +12972,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13315,7 +12988,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13339,7 +13011,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-08, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview", @@ -13352,7 +13023,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13365,7 +13035,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13377,7 +13046,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview", @@ -13389,7 +13057,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13403,7 +13070,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "supports_assistant_prefill": true, @@ -13416,7 +13082,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13428,7 +13093,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -13441,7 +13105,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -13455,7 +13118,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -13467,7 +13129,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13477,7 +13138,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13487,7 +13147,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13497,7 +13156,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_tool_choice": true }, @@ -13522,7 +13180,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13532,7 +13189,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/ap-northeast-1/1-month-commitment/anthropic.claude-v2:1": { @@ -13541,7 +13197,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13551,7 +13206,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13561,7 +13215,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/ap-northeast-1/6-month-commitment/anthropic.claude-v2:1": { @@ -13570,7 +13223,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -13579,7 +13231,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 7.55e-06, "supports_tool_choice": true @@ -13589,7 +13240,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -13599,7 +13249,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -13610,7 +13259,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -13630,7 +13278,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13648,7 +13295,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13668,7 +13314,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, "output_cost_per_token_flex": 1.52e-06, @@ -13682,7 +13327,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -13700,7 +13344,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13719,7 +13362,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -13733,7 +13375,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -13753,7 +13394,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -13764,7 +13404,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -13775,7 +13414,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -13795,7 +13433,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13813,7 +13450,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13833,7 +13469,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.94e-06, "output_cost_per_token_flex": 1.47e-06, @@ -13847,7 +13482,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -13865,7 +13499,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13883,7 +13516,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13902,7 +13534,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -13922,7 +13553,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13940,7 +13570,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -13959,7 +13588,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -13977,7 +13605,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -13995,7 +13622,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.03e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14006,7 +13632,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.9e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14017,7 +13642,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -14037,7 +13661,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14055,7 +13678,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14074,7 +13696,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -14092,7 +13713,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14102,7 +13722,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-v2:1": { @@ -14111,7 +13730,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14121,7 +13739,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14131,7 +13748,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/eu-central-1/6-month-commitment/anthropic.claude-v2:1": { @@ -14140,7 +13756,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14149,7 +13764,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 8.38e-06, "supports_tool_choice": true @@ -14159,7 +13773,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05 }, @@ -14168,7 +13781,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -14180,7 +13792,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14198,7 +13809,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14217,7 +13827,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14235,7 +13844,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.78e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14246,7 +13854,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14258,7 +13865,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14276,7 +13882,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14295,7 +13900,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14313,7 +13917,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.55e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14324,7 +13927,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14336,7 +13938,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, "output_cost_per_token_flex": 9.3e-07, @@ -14354,7 +13955,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14373,7 +13973,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.01e-06, "output_cost_per_token_flex": 5.05e-07, @@ -14392,7 +13991,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.86e-06, "output_cost_per_token_flex": 9.3e-07, @@ -14410,7 +14008,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -14421,7 +14018,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.56e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -14433,7 +14029,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9.1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -14446,7 +14041,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14464,7 +14058,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14483,7 +14076,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14500,7 +14092,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Anthropic via Invoke route does not currently support pdf input." }, @@ -14519,7 +14110,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.88e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14530,7 +14120,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.01e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14541,7 +14130,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -14561,7 +14149,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14579,7 +14166,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14599,7 +14185,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, "output_cost_per_token_flex": 1.52e-06, @@ -14613,7 +14198,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, @@ -14631,7 +14215,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, @@ -14649,7 +14232,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14659,7 +14241,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-v2:1": { @@ -14668,7 +14249,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14678,7 +14258,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14688,7 +14267,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-east-1/6-month-commitment/anthropic.claude-v2:1": { @@ -14697,7 +14275,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -14706,7 +14283,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-06, "supports_tool_choice": true @@ -14716,7 +14292,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -14726,7 +14301,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -14737,7 +14311,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -14748,7 +14321,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -14759,7 +14331,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -14770,7 +14341,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -14782,7 +14352,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -14794,7 +14363,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_flex": 9.25e-07, @@ -14814,7 +14382,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -14834,7 +14401,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -14851,7 +14417,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -14865,7 +14430,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -14883,7 +14447,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -14901,7 +14464,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_flex": 9.25e-07, @@ -14921,7 +14483,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -14939,7 +14500,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, @@ -14959,7 +14519,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -14973,7 +14532,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -14991,7 +14549,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -15009,7 +14566,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.84e-06, "supports_function_calling": true, @@ -15023,7 +14579,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -15032,7 +14587,6 @@ "input_cost_per_token": 2e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -15042,7 +14596,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.7e-06 }, @@ -15051,7 +14604,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-07 }, @@ -15060,7 +14612,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -15073,7 +14624,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15102,7 +14652,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15123,7 +14672,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.5e-06, "supports_pdf_input": true @@ -15133,7 +14681,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 2.65e-06, "supports_pdf_input": true @@ -15141,7 +14688,6 @@ "bedrock/us-gov-west-1/amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 8172, - "max_tokens": 8172, "mode": "embedding", "input_cost_per_token": 1.62e-07, "input_cost_per_image": 7.2e-05, @@ -15160,7 +14706,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.88e-07, "supports_function_calling": true, @@ -15176,7 +14721,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.68e-07, "supports_function_calling": true, @@ -15190,7 +14734,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.84e-06, "supports_function_calling": true, @@ -15204,7 +14747,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536 @@ -15213,7 +14755,6 @@ "input_cost_per_token": 1.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -15223,7 +14764,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.6e-06 }, @@ -15232,7 +14772,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-07 }, @@ -15241,7 +14780,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 42000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06 }, @@ -15254,7 +14792,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15283,7 +14820,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -15304,7 +14840,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 3.5e-06, "supports_pdf_input": true @@ -15314,7 +14849,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6e-07, "supports_pdf_input": true @@ -15325,7 +14859,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -15336,7 +14869,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -15347,7 +14879,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15357,7 +14888,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-west-2/1-month-commitment/anthropic.claude-v2:1": { @@ -15366,7 +14896,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15376,7 +14905,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15386,7 +14914,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat" }, "bedrock/us-west-2/6-month-commitment/anthropic.claude-v2:1": { @@ -15395,7 +14922,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "supports_tool_choice": true }, @@ -15404,7 +14930,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-06, "supports_tool_choice": true @@ -15414,7 +14939,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -15424,7 +14948,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 100000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.4e-05, "supports_tool_choice": true @@ -15435,7 +14958,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -15446,7 +14968,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -15458,7 +14979,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -15470,7 +14990,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_flex": 9.25e-07, @@ -15490,7 +15009,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -15510,7 +15028,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -15527,7 +15044,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_flex": 1.25e-06, @@ -15541,7 +15057,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_flex": 1.5e-06, @@ -15559,7 +15074,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, @@ -15578,7 +15092,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -15668,7 +15181,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -15679,7 +15191,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -15690,7 +15201,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true, @@ -15701,7 +15211,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -15712,7 +15221,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -15723,7 +15231,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.5e-07, "source": "https://www.cerebras.ai/blog/openai-gpt-oss-120b-runs-fastest-on-cerebras", @@ -15738,7 +15245,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://inference-docs.cerebras.ai/support/pricing", @@ -15751,7 +15257,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.49e-06, "source": "https://api.cerebras.ai/public/v1/models/qwen-3.8-27b", @@ -15767,7 +15272,6 @@ "litellm_provider": "nlp_cloud", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5e-07 }, @@ -15798,7 +15302,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -15825,7 +15328,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -15854,7 +15356,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -15888,7 +15389,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -15924,7 +15424,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -15968,7 +15467,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -16015,7 +15513,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -16059,7 +15556,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -16088,7 +15584,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16122,7 +15617,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16159,7 +15653,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16202,7 +15695,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16245,7 +15737,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16289,7 +15780,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16333,7 +15823,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -16378,7 +15867,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -16424,7 +15912,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1e-05, @@ -16470,7 +15957,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16518,7 +16004,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -16634,7 +16119,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 3072, "max_output_tokens": 3072, - "max_tokens": 3072, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16644,7 +16128,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 2048, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16654,7 +16137,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16664,7 +16146,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.923e-06, "rpm": 300 @@ -16674,7 +16155,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-07, "rpm": 300, @@ -16686,7 +16166,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16696,7 +16175,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 80000, "max_output_tokens": 80000, - "max_tokens": 80000, "mode": "chat", "output_cost_per_token": 3.35e-07, "rpm": 300 @@ -16706,7 +16184,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-08, "rpm": 300 @@ -16716,7 +16193,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 15000, "max_output_tokens": 15000, - "max_tokens": 15000, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16727,7 +16203,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "rpm": 20, @@ -16739,7 +16214,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 80000, "max_output_tokens": 80000, - "max_tokens": 80000, "mode": "chat", "output_cost_per_token": 4.881e-06, "rpm": 300, @@ -16750,7 +16224,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.87e-07, "rpm": 300 @@ -16760,7 +16233,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 60000, "max_output_tokens": 60000, - "max_tokens": 60000, "mode": "chat", "output_cost_per_token": 2.01e-07, "rpm": 300 @@ -16771,7 +16243,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "rpm": 20, @@ -16783,7 +16254,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-07, "rpm": 300, @@ -16795,7 +16265,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16805,7 +16274,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 24000, "max_output_tokens": 24000, - "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 2.253e-06, "rpm": 300, @@ -16816,7 +16284,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.12e-07, "rpm": 300, @@ -16827,7 +16294,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1e-06, "rpm": 300 @@ -16838,7 +16304,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "rpm": 20, @@ -16850,7 +16315,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-06, "rpm": 300, @@ -16862,7 +16326,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.55e-07, "rpm": 300 @@ -16872,7 +16335,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.35e-07, "rpm": 300, @@ -16884,7 +16346,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 3500, "max_output_tokens": 3500, - "max_tokens": 3500, "mode": "chat", "output_cost_per_token": 0.0, "rpm": 300 @@ -16894,7 +16355,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, "rpm": 300, @@ -16906,7 +16366,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.55e-07, "rpm": 300, @@ -16917,7 +16376,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.76e-07, "rpm": 300, @@ -16928,7 +16386,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "rpm": 300, @@ -16940,7 +16397,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 8.5e-07, "rpm": 300, @@ -16951,7 +16407,6 @@ "litellm_provider": "cloudflare", "max_input_tokens": 24000, "max_output_tokens": 24000, - "max_tokens": 24000, "mode": "chat", "output_cost_per_token": 1e-06, "rpm": 300, @@ -16962,7 +16417,6 @@ "litellm_provider": "codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/", @@ -16974,7 +16428,6 @@ "litellm_provider": "codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/", @@ -16986,7 +16439,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -16997,7 +16449,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17007,7 +16458,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17017,7 +16467,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "bedrock", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -17027,7 +16476,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17038,7 +16486,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17049,7 +16496,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17060,7 +16506,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "cohere", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -17072,7 +16517,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "cohere", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048, @@ -17083,7 +16527,6 @@ "input_cost_per_token": 8e-08, "litellm_provider": "cohere", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048, @@ -17105,7 +16548,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-cohere-rerank-3-5.md" @@ -17115,7 +16557,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -17127,7 +16568,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "source": "https://docs.cohere.com/docs/models" }, @@ -17136,7 +16576,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.cohere.com/docs/command-a-plus", @@ -17151,7 +16590,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06 }, @@ -17160,7 +16598,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -17171,7 +16608,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -17182,7 +16618,6 @@ "litellm_provider": "cohere_chat", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.cohere.com/v2/docs/command-r7b", @@ -17195,7 +16630,7 @@ "input_cost_per_token": 1.12e-06, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17218,7 +16653,7 @@ "input_cost_per_token": 1.12e-06, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17241,7 +16676,7 @@ "input_cost_per_token": 1.12e-06, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17264,7 +16699,7 @@ "input_cost_per_token": 1e-08, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17287,7 +16722,7 @@ "input_cost_per_token": 1e-08, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17310,7 +16745,7 @@ "input_cost_per_token": 1e-08, "litellm_provider": "coralbricks", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.coralbricks.ai/pricing", @@ -17333,7 +16768,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, "supported_endpoints": [ @@ -17363,7 +16797,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.2e-05, "supported_endpoints": [ @@ -17394,7 +16827,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -17416,7 +16848,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -17438,7 +16869,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17454,7 +16884,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17470,7 +16899,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4.8e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17486,7 +16914,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 202745, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17502,7 +16929,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17518,7 +16944,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 229376, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17534,7 +16959,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17546,7 +16970,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17575,7 +16998,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17605,7 +17027,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17618,7 +17039,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17631,7 +17051,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17644,7 +17063,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -17658,7 +17076,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -17671,7 +17088,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17702,7 +17118,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17733,7 +17148,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17765,7 +17179,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -17779,7 +17192,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -17792,7 +17204,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -17806,7 +17217,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -17819,7 +17229,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17830,7 +17239,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17879,7 +17287,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17924,7 +17331,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -17973,7 +17379,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18018,7 +17423,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18055,7 +17459,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18092,7 +17495,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18130,7 +17532,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18142,7 +17543,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18155,7 +17555,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18168,7 +17567,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18182,7 +17580,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18195,7 +17592,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.87e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -18208,7 +17604,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 260096, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18246,7 +17641,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18280,7 +17674,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -18294,7 +17687,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -18330,7 +17722,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -18348,7 +17739,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -18367,7 +17757,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -18386,7 +17775,6 @@ "litellm_provider": "dashscope", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -18707,7 +18095,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.qwencloud.com/models", @@ -18723,7 +18110,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.qwencloud.com/models", @@ -18739,7 +18125,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4.8e-06, "source": "https://www.qwencloud.com/models", @@ -18755,7 +18140,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 202745, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.qwencloud.com/models", @@ -18771,7 +18155,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.qwencloud.com/models", @@ -18787,7 +18170,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 229376, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.qwencloud.com/models", @@ -18803,7 +18185,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.qwencloud.com/models", @@ -18815,7 +18196,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -18844,7 +18224,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -18874,7 +18253,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.qwencloud.com/models", @@ -18887,7 +18265,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -18900,7 +18277,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -18913,7 +18289,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -18927,7 +18302,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -18940,7 +18314,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -18971,7 +18344,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19002,7 +18374,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19034,7 +18405,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -19048,7 +18418,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.qwencloud.com/models", @@ -19061,7 +18430,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -19075,7 +18443,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -19088,7 +18455,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19099,7 +18465,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19148,7 +18513,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19193,7 +18557,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19242,7 +18605,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19287,7 +18649,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19324,7 +18685,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19361,7 +18721,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19399,7 +18758,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -19411,7 +18769,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.qwencloud.com/models", @@ -19424,7 +18781,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://www.qwencloud.com/models", @@ -19437,7 +18793,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.qwencloud.com/models", @@ -19451,7 +18806,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://www.qwencloud.com/models", @@ -19464,7 +18818,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.87e-06, "source": "https://www.qwencloud.com/models", @@ -19477,7 +18830,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 260096, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19515,7 +18867,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19549,7 +18900,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://www.qwencloud.com/models", @@ -19563,7 +18913,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.qwencloud.com/models", "supports_function_calling": true, @@ -19599,7 +18948,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://www.qwencloud.com/models", @@ -19615,7 +18963,6 @@ "litellm_provider": "qwencloud", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.qwencloud.com/models", @@ -19936,7 +19283,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -19952,7 +19298,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -19968,7 +19313,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 4.8e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -19984,7 +19328,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 202745, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20000,7 +19343,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20016,7 +19358,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 229376, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20032,7 +19373,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20044,7 +19384,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20073,7 +19412,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20103,7 +19441,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 30720, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20116,7 +19453,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20129,7 +19465,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20142,7 +19477,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -20156,7 +19490,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, @@ -20169,7 +19502,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20200,7 +19532,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20231,7 +19562,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20263,7 +19593,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -20277,7 +19606,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20290,7 +19618,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -20304,7 +19631,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 1000000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_reasoning_token": 5e-07, "output_cost_per_token": 2e-07, @@ -20317,7 +19643,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 129024, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20328,7 +19653,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20377,7 +19701,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20422,7 +19745,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20471,7 +19793,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 997952, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20516,7 +19837,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20553,7 +19873,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20590,7 +19909,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 258048, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20628,7 +19946,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20640,7 +19957,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20653,7 +19969,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20666,7 +19981,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20680,7 +19994,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6.4e-07, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20693,7 +20006,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.87e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", @@ -20706,7 +20018,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 260096, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20744,7 +20055,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20778,7 +20088,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20792,7 +20101,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supports_function_calling": true, @@ -20828,7 +20136,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -20846,7 +20153,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -20865,7 +20171,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 991808, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://docs.modelstudio.console.alibabacloud.com/en/model-studio/model-pricing", @@ -20884,7 +20189,6 @@ "litellm_provider": "qwen_ai_platform", "max_input_tokens": 98304, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://www.alibabacloud.com/help/en/model-studio/models", @@ -21206,7 +20510,6 @@ "input_dbu_cost_per_token": 1.429e-06, "litellm_provider": "databricks", "max_input_tokens": 512, - "max_tokens": 512, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21224,7 +20527,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21253,7 +20555,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21283,7 +20584,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21307,7 +20607,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21331,7 +20630,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21355,7 +20653,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21380,7 +20677,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21405,7 +20701,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21432,7 +20727,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21460,7 +20754,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21489,7 +20782,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -21524,7 +20816,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21548,7 +20839,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21571,7 +20861,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21595,7 +20884,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21620,7 +20908,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields. Introductory launch rates of 28.571 input / 142.857 output / 35.714 cache write / 2.857 cache read DBU run through 2026-08-31; the standard rates are listed here because entries carry no expiry date." }, @@ -21649,7 +20936,6 @@ "token_count_framing": 97, "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference. Context/max output are the DeepSeek-published model limits (1M context, 384K max output)." }, @@ -21682,7 +20968,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference. Context/max output are the DeepSeek-published model limits (1M context, 384K max output)." }, @@ -21711,7 +20996,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21745,7 +21029,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21780,7 +21063,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21808,7 +21090,6 @@ "litellm_provider": "databricks", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21829,7 +21110,6 @@ "litellm_provider": "databricks", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21854,7 +21134,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21887,7 +21166,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21919,7 +21197,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -21935,7 +21212,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21960,7 +21236,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Databricks DBU rates not yet published for this model; endpoint metadata only." }, @@ -21989,7 +21264,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22021,7 +21295,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22052,7 +21325,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22085,7 +21357,6 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22102,7 +21373,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference." }, @@ -22130,7 +21400,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22161,7 +21430,6 @@ "litellm_provider": "databricks", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22191,7 +21459,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22211,7 +21478,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22231,7 +21497,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22251,7 +21516,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22271,7 +21535,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22300,7 +21563,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22329,7 +21591,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22358,7 +21619,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields. Rates reflect OpenAI's promotional pricing in effect through November 21, 2026; afterwards input, cache and Batch rates are 25% higher and output rates 50% higher." }, @@ -22387,7 +21647,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22416,7 +21675,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22445,7 +21703,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22474,7 +21731,6 @@ "litellm_provider": "databricks", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22503,7 +21759,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22523,7 +21778,6 @@ "litellm_provider": "databricks", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22543,7 +21797,6 @@ "litellm_provider": "databricks", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22560,7 +21813,6 @@ "litellm_provider": "databricks", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22602,7 +21854,6 @@ "input_dbu_cost_per_token": 1.857e-06, "litellm_provider": "databricks", "max_input_tokens": 8192, - "max_tokens": 8192, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22649,7 +21900,6 @@ "token_count_framing": 100, "max_input_tokens": 1000000, "max_output_tokens": 1048576, - "max_tokens": 1048576, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Billing reads the per-token dollar fields; the '*_dbu_cost_per_token' fields are the published Databricks rates, kept for reference." }, @@ -22683,7 +21933,6 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)." }, @@ -22701,7 +21950,6 @@ "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22718,7 +21966,6 @@ "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "metadata": { "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, @@ -22736,7 +21983,6 @@ "litellm_provider": "databricks", "max_input_tokens": 262144, "max_output_tokens": 25000, - "max_tokens": 25000, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22786,7 +22032,6 @@ "input_dbu_cost_per_token": 2.86e-07, "litellm_provider": "databricks", "max_input_tokens": 32768, - "max_tokens": 32768, "metadata": { "notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields." }, @@ -22832,7 +22077,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -23427,7 +22671,6 @@ ] }, "deepinfra/Gryphe/MythoMax-L2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 4e-07, @@ -23439,7 +22682,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/NousResearch/Hermes-3-Llama-3.1-405B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-06, @@ -23450,7 +22692,6 @@ "supports_function_calling": true }, "deepinfra/NousResearch/Hermes-3-Llama-3.1-70B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 7e-07, @@ -23461,7 +22702,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/QwQ-32B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.5e-07, @@ -23472,7 +22712,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen2.5-72B-Instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 3.6e-07, @@ -23484,7 +22723,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen2.5-7B-Instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 4e-08, @@ -23494,7 +22732,6 @@ "supports_tool_choice": false }, "deepinfra/Qwen/Qwen2.5-VL-32B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -23506,7 +22743,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-14B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1.2e-07, @@ -23518,7 +22754,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-235B-A22B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1.8e-07, @@ -23529,7 +22764,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-08, @@ -23541,7 +22775,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 3e-07, @@ -23552,7 +22785,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-30B-A3B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1.2e-07, @@ -23564,7 +22796,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-32B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 8e-08, @@ -23576,7 +22807,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 4e-07, @@ -23587,7 +22817,6 @@ "supports_function_calling": true }, "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct-Turbo": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 3e-07, @@ -23601,7 +22830,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Next-80B-A3B-Instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-08, @@ -23613,7 +22841,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Next-80B-A3B-Thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.4e-07, @@ -23624,7 +22851,6 @@ "supports_function_calling": true }, "deepinfra/Sao10K/L3-8B-Lunaris-v1-Turbo": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 4e-08, @@ -23634,7 +22860,6 @@ "supports_tool_choice": false }, "deepinfra/Sao10K/L3.1-70B-Euryale-v2.2": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 8.5e-07, @@ -23645,7 +22870,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Sao10K/L3.3-70B-Euryale-v2.3": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6.5e-07, @@ -23655,7 +22879,6 @@ "supports_tool_choice": false }, "deepinfra/allenai/olmOCR-7B-0725-FP8": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2.7e-07, @@ -23665,7 +22888,6 @@ "supports_tool_choice": false }, "deepinfra/anthropic/claude-3-7-sonnet-latest": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 3.3e-06, @@ -23677,7 +22899,6 @@ "supports_function_calling": true }, "deepinfra/anthropic/claude-4-opus": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 1.65e-05, @@ -23688,7 +22909,6 @@ "supports_function_calling": true }, "deepinfra/anthropic/claude-4-sonnet": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 3.3e-06, @@ -23699,7 +22919,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 7e-07, @@ -23710,7 +22929,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-0528": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -23722,7 +22940,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-0528-Turbo": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-06, @@ -23733,7 +22950,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -23743,7 +22959,6 @@ "supports_tool_choice": false }, "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.7e-07, @@ -23754,7 +22969,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-R1-Turbo": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-06, @@ -23765,7 +22979,6 @@ "supports_function_calling": true }, "deepinfra/deepseek-ai/DeepSeek-V3": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 3.2e-07, @@ -23777,7 +22990,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3-0324": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 2.4e-07, @@ -23791,7 +23003,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.1": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 2.5e-07, @@ -23805,7 +23016,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.1-Terminus": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 2.7e-07, @@ -23817,7 +23027,6 @@ "supports_function_calling": true }, "deepinfra/google/gemini-2.5-flash": { - "max_tokens": 1000000, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "input_cost_per_token": 3e-07, @@ -23833,7 +23042,6 @@ "supports_audio_input": true }, "deepinfra/google/gemini-2.5-pro": { - "max_tokens": 1000000, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "input_cost_per_token": 1.25e-06, @@ -23848,7 +23056,6 @@ "supports_audio_input": true }, "deepinfra/google/gemma-3-12b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-08, @@ -23860,7 +23067,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-3-27b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 8e-08, @@ -23872,7 +23078,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-3-4b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-08, @@ -23884,7 +23089,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-3.2-11B-Vision-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4.9e-08, @@ -23894,7 +23098,6 @@ "supports_tool_choice": false }, "deepinfra/meta-llama/Llama-3.2-3B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, @@ -23905,7 +23108,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.3e-07, @@ -23916,7 +23118,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -23928,7 +23129,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 2e-07, @@ -23940,7 +23140,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "max_tokens": 327680, "max_input_tokens": 327680, "max_output_tokens": 327680, "input_cost_per_token": 1e-07, @@ -23952,7 +23151,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Llama-Guard-3-8B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5.5e-08, @@ -23962,7 +23160,6 @@ "supports_tool_choice": false }, "deepinfra/meta-llama/Llama-Guard-4-12B": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1.8e-07, @@ -23972,7 +23169,6 @@ "supports_tool_choice": false }, "deepinfra/meta-llama/Meta-Llama-3-8B-Instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 3e-08, @@ -23983,7 +23179,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-07, @@ -23994,7 +23189,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-07, @@ -24006,7 +23200,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -24017,7 +23210,6 @@ "supports_function_calling": true }, "deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, @@ -24029,7 +23221,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/microsoft/WizardLM-2-8x22B": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 4.8e-07, @@ -24039,7 +23230,6 @@ "supports_tool_choice": false }, "deepinfra/microsoft/phi-4": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 7e-08, @@ -24050,7 +23240,6 @@ "supports_function_calling": true }, "deepinfra/mistralai/Mistral-Nemo-Instruct-2407": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.9e-08, @@ -24062,7 +23251,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/mistralai/Mistral-Small-24B-Instruct-2501": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-08, @@ -24073,7 +23261,6 @@ "supports_function_calling": true }, "deepinfra/mistralai/Mistral-Small-3.2-24B-Instruct-2506": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 7.5e-08, @@ -24084,7 +23271,6 @@ "supports_function_calling": true }, "deepinfra/mistralai/Mixtral-8x7B-Instruct-v0.1": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 4e-07, @@ -24095,7 +23281,6 @@ "supports_function_calling": true }, "deepinfra/moonshotai/Kimi-K2-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-07, @@ -24106,7 +23291,6 @@ "supports_function_calling": true }, "deepinfra/moonshotai/Kimi-K2-Instruct-0905": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 5e-07, @@ -24118,7 +23302,6 @@ "supports_function_calling": true }, "deepinfra/nvidia/Llama-3.1-Nemotron-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6e-07, @@ -24129,7 +23312,6 @@ "supports_function_calling": true }, "deepinfra/nvidia/Llama-3.3-Nemotron-Super-49B-v1.5": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -24153,7 +23335,6 @@ "supports_prompt_caching": true }, "deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-08, @@ -24164,7 +23345,6 @@ "supports_function_calling": true }, "deepinfra/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3.7e-08, @@ -24176,7 +23356,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/openai/gpt-oss-20b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -24188,7 +23367,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.5": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 4e-07, @@ -24207,7 +23385,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -24230,7 +23407,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07, "supports_assistant_prefill": true, @@ -24245,7 +23421,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.19e-06, "supports_assistant_prefill": true, @@ -24262,7 +23437,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", @@ -24287,7 +23461,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.1e-06, "supports_assistant_prefill": true, @@ -24302,7 +23475,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 4e-07, "supports_assistant_prefill": true, @@ -24317,7 +23489,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-1.html", @@ -24334,7 +23505,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 164000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_batches": 9.25e-07, @@ -24354,7 +23524,6 @@ "litellm_provider": "nlp_cloud", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "completion", "output_cost_per_token": 5e-07 }, @@ -24363,7 +23532,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 98304, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_assistant_prefill": true, @@ -24377,7 +23545,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "supports_assistant_prefill": true, @@ -24391,7 +23558,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 229376, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_assistant_prefill": true, @@ -24404,7 +23570,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - standard version with 2560 dimensions" }, @@ -24416,7 +23581,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - large version with 2048 dimensions" }, @@ -24428,7 +23592,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - text-240915 version with 4096 dimensions" }, @@ -24440,7 +23603,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - text-250515 version with 2048 dimensions" }, @@ -24452,7 +23614,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "volcengine", "max_input_tokens": 4096, - "max_tokens": 4096, "metadata": { "notes": "Volcengine Doubao embedding model - text-240715 version with 2560 dimensions" }, @@ -24565,7 +23726,6 @@ "mode": "search" }, "sail/moonshotai/Kimi-K3": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 2.5e-06, @@ -24587,7 +23747,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/zai-org/GLM-5.3": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9.8e-07, @@ -24609,7 +23768,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/zai-org/GLM-5.3-Flash": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.1e-07, @@ -24631,7 +23789,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/deepseek-ai/DeepSeek-V4-Pro-0813": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9.2e-07, @@ -24653,7 +23810,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9e-08, @@ -24675,7 +23831,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/deepseek-ai/DeepSeek-V4.1-Flash": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 9e-08, @@ -24697,7 +23852,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/moonshotai/Kimi-K2.6": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-06, @@ -24720,7 +23874,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/google/gemma-4-31B-it": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 4e-07, @@ -24743,7 +23896,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/nvidia/Gemma-4-31B-IT-NVFP4": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.4e-07, @@ -24766,7 +23918,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/google/gemma-4-12B-it": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 3e-07, @@ -24788,7 +23939,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6e-08, @@ -24804,7 +23954,6 @@ "source": "https://docs.sailresearch.com/models" }, "sail/Qwen/Qwen3.6-35B-A3B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 5e-08, @@ -25043,7 +24192,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -25052,7 +24200,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "metadata": { "notes": "'supports_image_input' is a deprecated field. Use 'supports_embedding_image_input' instead." }, @@ -25065,7 +24212,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "supports_embedding_image_input": true @@ -25074,7 +24220,6 @@ "input_cost_per_token": 0.0001, "litellm_provider": "cohere", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "supports_embedding_image_input": true @@ -25085,7 +24230,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.12e-07, "supports_function_calling": true, @@ -25101,7 +24245,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.84e-07, "supports_function_calling": true, @@ -25117,7 +24260,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 4.2e-06, "output_cost_per_token_flex": 2.1e-06, @@ -25134,7 +24276,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -25157,7 +24298,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -25185,7 +24325,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -25201,7 +24340,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -25220,7 +24358,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -25240,7 +24377,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -25257,7 +24393,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -25288,7 +24423,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -25319,7 +24453,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -25359,7 +24492,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -25393,7 +24525,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.3e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", @@ -25406,7 +24537,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", @@ -25418,7 +24548,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md", @@ -27684,14 +26813,12 @@ "litellm_provider": "featherless_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat" }, "featherless_ai/featherless-ai/Qwerky-QwQ-32B": { "litellm_provider": "featherless_ai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat" }, "fireworks-ai-4.1b-to-16b": { @@ -27739,7 +26866,6 @@ "input_cost_per_token": 1.6e-08, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -27749,7 +26875,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/pricing", @@ -27762,7 +26887,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 20480, - "max_tokens": 20480, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://fireworks.ai/pricing", @@ -27774,7 +26898,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 160000, - "max_tokens": 160000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://fireworks.ai/pricing", @@ -27786,7 +26909,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 20480, - "max_tokens": 20480, "mode": "chat", "output_cost_per_token": 2.19e-06, "source": "https://fireworks.ai/pricing", @@ -27798,7 +26920,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -27810,7 +26931,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/models/fireworks/deepseek-v3-0324", @@ -27822,7 +26942,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://fireworks.ai/pricing", @@ -27835,7 +26954,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://fireworks.ai/pricing", @@ -27848,7 +26966,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.68e-06, "source": "https://fireworks.ai/models/fireworks/deepseek-v3p2", @@ -27863,7 +26980,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -27882,7 +26998,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.96e-06, "output_cost_per_token_priority": 4.95e-06, @@ -27902,7 +27017,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.96e-06, "output_cost_per_token_priority": 4.95e-06, @@ -27918,7 +27032,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -27931,7 +27044,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 96000, - "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 2.19e-06, "source": "https://fireworks.ai/models/fireworks/glm-4p5", @@ -27945,7 +27057,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 96000, - "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://artificialanalysis.ai/models/glm-4-5-air", @@ -27960,7 +27071,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "source": "https://fireworks.ai/pricing", "supports_function_calling": true, @@ -27974,7 +27084,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://fireworks.ai/models/fireworks/glm-4p7", @@ -27989,7 +27098,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28008,7 +27116,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -28028,7 +27135,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 7.2e-07, @@ -28045,7 +27151,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28060,7 +27165,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct", @@ -28073,7 +27177,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://app.fireworks.ai/models/fireworks/kimi-k2-instruct-0905", @@ -28086,7 +27189,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://fireworks.ai/pricing", @@ -28101,7 +27203,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28118,7 +27219,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28138,7 +27238,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28154,7 +27253,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28167,7 +27265,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://fireworks.ai/pricing", @@ -28180,7 +27277,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://fireworks.ai/pricing", @@ -28194,7 +27290,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://fireworks.ai/pricing", @@ -28207,7 +27302,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://fireworks.ai/pricing", @@ -28220,7 +27314,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -28233,7 +27326,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://fireworks.ai/pricing", @@ -28245,7 +27337,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://fireworks.ai/pricing", @@ -28258,7 +27349,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 196608, "max_output_tokens": 204800, - "max_tokens": 204800, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/models/fireworks/minimax-m2p1", @@ -28274,7 +27364,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.8e-06, @@ -28290,7 +27379,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/pricing", @@ -28303,7 +27391,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -28316,7 +27403,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://fireworks.ai/pricing", @@ -28329,7 +27415,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28343,7 +27428,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28359,7 +27443,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://fireworks.ai/models/fireworks/glm-4p7", @@ -28374,7 +27457,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202752, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28390,7 +27472,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28409,7 +27490,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -28429,7 +27509,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 7.2e-07, @@ -28446,7 +27525,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28462,7 +27540,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://fireworks.ai/pricing", @@ -28479,7 +27556,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28497,7 +27573,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28516,7 +27591,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-06, "output_cost_per_token_priority": 6e-06, @@ -28534,7 +27608,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -28550,7 +27623,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 196608, "max_output_tokens": 204800, - "max_tokens": 204800, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://fireworks.ai/models/fireworks/minimax-m2p1", @@ -28566,7 +27638,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.8e-06, @@ -28583,7 +27654,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -28597,7 +27667,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28606,7 +27675,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28615,7 +27683,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28624,7 +27691,6 @@ "input_cost_per_token": 1.6e-08, "litellm_provider": "fireworks_ai-embedding-models", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "source": "https://fireworks.ai/pricing" @@ -28633,7 +27699,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 3e-08, @@ -28661,7 +27726,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "input_cost_per_token": 1.26e-06, "output_cost_per_token": 3.96e-06, "cache_read_input_token_cost": 2.34e-07, @@ -28689,7 +27753,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 4e-07, "supports_prompt_caching": false, @@ -28712,7 +27775,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, @@ -28739,7 +27801,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "input_cost_per_token": 5e-07, "output_cost_per_token": 1.5e-06, "cache_read_input_token_cost": 2.5e-07, @@ -28763,7 +27824,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 196608, "max_output_tokens": 196608, - "max_tokens": 196608, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-08, @@ -28787,7 +27847,6 @@ "litellm_provider": "friendliai", "max_input_tokens": 202752, "max_output_tokens": 202752, - "max_tokens": 202752, "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, @@ -28814,7 +27873,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 9e-07, @@ -28827,7 +27885,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, @@ -28840,7 +27897,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_batches": 3e-06, @@ -28854,7 +27910,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_system_messages": true, @@ -28866,7 +27921,6 @@ "litellm_provider": "openai", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_system_messages": true, @@ -28878,7 +27932,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-06, "supports_system_messages": true, @@ -28890,7 +27943,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "source": "OpenAI needs to add pricing for this ft model, will be updated when added by OpenAI. Defaulting to base model pricing", @@ -28907,7 +27959,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -28928,7 +27979,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -28948,7 +27998,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -28970,7 +28019,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, @@ -28991,7 +28039,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.2e-06, "output_cost_per_token_batches": 1.6e-06, @@ -29012,7 +28059,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, "output_cost_per_token_batches": 4e-07, @@ -29033,7 +28079,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 1.6e-05, "output_cost_per_token_batches": 8e-06, @@ -29056,7 +28101,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29113,7 +28157,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, @@ -29171,7 +28214,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -29225,7 +28267,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -29277,7 +28318,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -29323,7 +28363,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -29372,7 +28411,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -29412,7 +28450,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -29471,7 +28508,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -29531,7 +28567,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29587,7 +28622,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -29625,7 +28659,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -29680,7 +28713,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -29727,7 +28759,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -29774,7 +28805,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -29820,7 +28850,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -29864,7 +28893,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -29916,7 +28944,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -29978,7 +29005,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_image": 0.00012, "output_cost_per_token": 1.2e-05, @@ -30031,7 +29057,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -30084,7 +29109,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -30145,7 +29169,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -30192,7 +29215,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -30255,7 +29277,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -30316,7 +29337,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -30380,7 +29400,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -30436,7 +29455,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 7.5e-06, "output_cost_per_token": 7.5e-06, @@ -30484,7 +29502,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -30527,7 +29544,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_image": 0.00012, "output_cost_per_token": 1.2e-05, @@ -30580,7 +29596,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -30637,7 +29652,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -30670,7 +29684,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_reasoning_token": 2.5e-06, @@ -30708,7 +29721,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, @@ -30755,7 +29767,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -30779,7 +29790,6 @@ "input_cost_per_token_batches": 1.2e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30796,7 +29806,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30814,7 +29823,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30833,7 +29841,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30852,7 +29859,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30866,7 +29872,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30878,7 +29883,6 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "gemini", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30897,7 +29901,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -30918,7 +29921,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -30964,7 +29966,6 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "image_generation", "output_cost_per_image": 0.045, "output_cost_per_image_token": 6e-05, @@ -31016,7 +30017,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -31076,7 +30076,6 @@ "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -31093,7 +30092,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "rpm": 1000, @@ -31130,7 +30128,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "rpm": 1000, @@ -31176,7 +30173,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -31239,7 +30235,6 @@ "supports_reasoning": false, "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, @@ -31291,7 +30286,6 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -31340,7 +30334,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -31383,7 +30376,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.045, "output_cost_per_image_token": 6e-05, @@ -31425,7 +30417,6 @@ "litellm_provider": "gemini", "max_input_tokens": 65536, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -31464,7 +30455,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, @@ -31509,7 +30499,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 4e-07, "output_cost_per_token": 4e-07, @@ -31566,7 +30555,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -31625,7 +30613,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -31684,7 +30671,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, @@ -31714,7 +30700,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -31768,7 +30753,6 @@ "litellm_provider": "gemini", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, @@ -31811,7 +30795,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -31873,7 +30856,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -31936,7 +30918,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -31998,7 +30979,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32063,7 +31043,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32125,7 +31104,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32188,7 +31166,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32246,7 +31223,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32284,7 +31260,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -32348,7 +31323,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -32417,7 +31391,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 3e-06, @@ -32462,7 +31435,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32497,7 +31469,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -32560,7 +31531,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32621,7 +31591,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32685,7 +31654,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -32741,7 +31709,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 7.5e-06, "output_cost_per_token": 7.5e-06, @@ -32788,7 +31755,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -32823,7 +31789,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "metadata": { "notes": "Rate limits not documented for gemini-exp-1114. Assuming same as gemini-1.5-pro.", "supports_tool_choice": true @@ -32846,7 +31811,6 @@ "litellm_provider": "gemini", "max_input_tokens": 2097152, "max_output_tokens": 8192, - "max_tokens": 8192, "metadata": { "notes": "Rate limits not documented for gemini-exp-1206. Assuming same as gemini-1.5-pro.", "supports_tool_choice": true @@ -32867,7 +31831,7 @@ "input_cost_per_token": 3.5e-07, "litellm_provider": "gemini", "max_output_tokens": 8192, - "max_tokens": 8192, + "max_input_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.05e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -32881,7 +31845,7 @@ "input_cost_per_token": 3.5e-07, "litellm_provider": "gemini", "max_output_tokens": 8192, - "max_tokens": 8192, + "max_input_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.05e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -32905,7 +31869,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_character": 0, "output_cost_per_character_above_128k_tokens": 0, @@ -32925,7 +31888,6 @@ "litellm_provider": "gemini", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -32941,7 +31903,6 @@ "litellm_provider": "gemini", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -32977,7 +31938,6 @@ "litellm_provider": "gemini", "max_input_tokens": 32767, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_character": 0, "output_cost_per_character_above_128k_tokens": 0, @@ -32996,7 +31956,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_image": 0.04, "output_cost_per_token": 0, @@ -33021,7 +31980,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -33045,7 +32003,6 @@ "deprecation_date": "2026-10-22", "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -33062,7 +32019,6 @@ "deprecation_date": "2026-10-22", "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -33078,7 +32034,6 @@ "deprecation_date": "2026-10-22", "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.05, "output_cost_per_second_1080p": 0.08, @@ -33093,7 +32048,6 @@ "gemini/veo-3.1-fast-generate-001": { "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -33109,7 +32063,6 @@ "gemini/veo-3.1-generate-001": { "litellm_provider": "gemini", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -33128,7 +32081,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33151,7 +32103,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33174,7 +32125,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 5e-06, "supported_endpoints": [ @@ -33195,7 +32145,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06, @@ -33220,7 +32169,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33243,7 +32191,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33266,7 +32213,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33288,7 +32234,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33311,7 +32256,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33333,7 +32277,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33356,7 +32299,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33375,7 +32317,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33392,7 +32333,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33400,7 +32340,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33408,7 +32347,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33416,7 +32354,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true }, @@ -33424,7 +32361,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33434,7 +32370,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "supports_function_calling": true, @@ -33447,7 +32382,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/model-deprecation-history.yml", "supports_function_calling": true, @@ -33463,7 +32397,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -33473,7 +32406,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -33483,7 +32415,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33492,7 +32423,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, @@ -33502,7 +32432,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33511,7 +32440,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 64000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true @@ -33523,7 +32451,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://github.blog/changelog/2026-10-19-github-copilot-models-deprecated-on-october-19-2026", @@ -33538,7 +32465,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "supported_endpoints": [ @@ -33559,7 +32485,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -33581,7 +32506,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 4.5e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33603,7 +32527,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -33627,7 +32550,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_200k_tokens": 1.8e-06, @@ -33651,7 +32573,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -33676,7 +32597,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -33698,9 +32618,8 @@ "input_cost_per_token": 1e-05, "input_cost_per_token_above_272k_tokens": 2e-05, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -33722,9 +32641,8 @@ "input_cost_per_token": 1e-07, "input_cost_per_token_above_272k_tokens": 2e-07, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 5e-07, "output_cost_per_token_above_272k_tokens": 7.5e-07, @@ -33746,9 +32664,8 @@ "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -33770,9 +32687,8 @@ "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "litellm_provider": "github_copilot", - "max_input_tokens": 272000, + "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -33792,7 +32708,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 917504, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33810,7 +32725,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.2e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -33825,19 +32739,16 @@ "github_copilot/text-embedding-3-small": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding" }, "github_copilot/text-embedding-3-small-inference": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding" }, "github_copilot/text-embedding-ada-002": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding" }, "chatgpt/gpt-6-sol": { @@ -33849,7 +32760,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33870,7 +32780,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33891,7 +32800,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33912,7 +32820,6 @@ ], "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "source": "https://developers.openai.com/api/docs/pricing", "supports_function_calling": true, "supports_max_reasoning_effort": true, @@ -33929,7 +32836,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.5", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -33949,7 +32855,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.6-luna", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -33969,7 +32874,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.6-sol", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -33989,7 +32893,6 @@ "source": "https://platform.openai.com/docs/models/gpt-5.6-terra", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34008,7 +32911,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34028,7 +32930,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34046,7 +32947,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34064,7 +32964,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34078,7 +32977,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34093,7 +32991,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34109,7 +33006,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34124,7 +33020,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/chat/completions", @@ -34144,7 +33039,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34159,7 +33053,6 @@ "litellm_provider": "chatgpt", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "responses", "supported_endpoints": [ "/v1/responses" @@ -34175,7 +33068,6 @@ "litellm_provider": "gigachat", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true, @@ -34186,7 +33078,6 @@ "litellm_provider": "gigachat", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true, @@ -34198,7 +33089,6 @@ "litellm_provider": "gigachat", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true, @@ -34209,7 +33099,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -34218,7 +33107,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024 @@ -34227,7 +33115,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 4096, - "max_tokens": 4096, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2560 @@ -34236,7 +33123,6 @@ "input_cost_per_token": 0.0, "litellm_provider": "gigachat", "max_input_tokens": 4096, - "max_tokens": 4096, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048 @@ -34246,7 +33132,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_function_calling": true, @@ -34258,7 +33143,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -34269,7 +33153,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -34280,7 +33163,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -34291,7 +33173,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.4e-05, "supports_function_calling": true @@ -34301,7 +33182,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true @@ -34311,7 +33191,6 @@ "litellm_provider": "gmi", "max_input_tokens": 409600, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true @@ -34321,7 +33200,6 @@ "litellm_provider": "gmi", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -34332,7 +33210,6 @@ "litellm_provider": "gmi", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -34343,7 +33220,6 @@ "litellm_provider": "gmi", "max_input_tokens": 163840, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-07, "supports_function_calling": true @@ -34353,7 +33229,6 @@ "litellm_provider": "gmi", "max_input_tokens": 163840, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 8.8e-07, "supports_function_calling": true @@ -34363,7 +33238,6 @@ "litellm_provider": "gmi", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -34375,7 +33249,6 @@ "litellm_provider": "gmi", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -34384,7 +33257,6 @@ "litellm_provider": "gmi", "max_input_tokens": 196608, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -34416,7 +33288,6 @@ "litellm_provider": "baseten", "max_input_tokens": 200000, "max_output_tokens": 200000, - "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -34461,7 +33332,6 @@ "litellm_provider": "baseten", "max_input_tokens": 128072, "max_output_tokens": 128072, - "max_tokens": 128072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inference.baseten.co/v1/models", @@ -34491,7 +33361,6 @@ "litellm_provider": "gmi", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-06, "supports_vision": true @@ -34501,7 +33370,6 @@ "litellm_provider": "gmi", "max_input_tokens": 202752, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2e-06 }, @@ -34514,7 +33382,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.9e-07, "output_cost_per_token_batches": 1.5e-07, @@ -34536,7 +33403,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 3.8e-07, "output_cost_per_token_batches": 1.9e-07, @@ -34558,7 +33424,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-08, "output_cost_per_token_batches": 4e-08, @@ -34590,7 +33455,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -34630,7 +33494,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -34664,7 +33527,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -34696,7 +33558,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -34717,7 +33578,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://developers.openai.com/api/docs/pricing", @@ -34733,7 +33593,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, @@ -34751,7 +33610,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 2e-06, @@ -34768,7 +33626,6 @@ "litellm_provider": "openai", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 4e-06, "supports_prompt_caching": true, @@ -34781,7 +33638,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 2e-06, "source": "https://developers.openai.com/api/docs/pricing" @@ -34791,7 +33647,6 @@ "litellm_provider": "text-completion-openai", "max_input_tokens": 8192, "max_output_tokens": 4097, - "max_tokens": 4097, "mode": "completion", "output_cost_per_token": 2e-06 }, @@ -34801,7 +33656,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "supports_function_calling": true, @@ -34816,7 +33670,6 @@ "litellm_provider": "openai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -34832,7 +33685,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -34847,7 +33699,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_function_calling": true, @@ -34865,7 +33716,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_batches": 1.5e-05, @@ -34887,7 +33737,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -34931,7 +33780,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_priority": 1.4e-05, @@ -34974,7 +33822,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 8e-07, @@ -35018,7 +33865,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "output_cost_per_token_priority": 2.8e-06, @@ -35062,7 +33908,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -35100,7 +33945,6 @@ "litellm_provider": "openai", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_priority": 8e-07, @@ -35137,7 +33981,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -35160,7 +34003,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -35184,7 +34026,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 1.7e-05, @@ -35209,7 +34050,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 1.7e-05, @@ -35231,7 +34071,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -35249,7 +34088,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 8e-05, "output_cost_per_token": 1e-05, @@ -35267,7 +34105,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -35304,7 +34141,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -35339,7 +34175,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1e-05, @@ -35377,7 +34212,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35415,7 +34249,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35452,7 +34285,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35493,7 +34325,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -35523,7 +34354,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 1e-06, @@ -35550,7 +34380,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 6e-07, @@ -35570,7 +34399,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -35588,7 +34416,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -35615,7 +34442,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -35696,7 +34522,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -35723,7 +34548,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -36277,7 +35101,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -36323,7 +35146,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5-codex", @@ -36358,7 +35180,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -36408,7 +35229,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-06, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex-mini", @@ -36439,7 +35259,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex-max", @@ -36471,7 +35290,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-codex", @@ -36503,7 +35321,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.1-chat-latest", @@ -36538,7 +35355,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_priority": 2e-05, @@ -36590,7 +35406,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -36641,7 +35456,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.2-codex", @@ -36673,7 +35487,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.2-chat-latest", @@ -36708,7 +35521,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -36758,7 +35570,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -36800,7 +35611,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.000168, "output_cost_per_token_batches": 8.4e-05, @@ -36840,10 +35650,14 @@ "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_12hr": 2.5e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 2.5e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": 0.0003, + "cache_creation_input_token_cost_above_12hr_batches": 1.25e-05, "cache_creation_input_token_cost_above_12hr_flex": 1.25e-05, "cache_creation_input_token_cost_above_12hr_priority": 5e-05, + "cache_creation_input_token_cost_above_12hr_ultrafast": 0.00015, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, @@ -36876,7 +35690,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, @@ -36931,8 +35744,10 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-06, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -36961,7 +35776,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -37014,8 +35828,10 @@ "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_12hr": 2.5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 2.5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06, + "cache_creation_input_token_cost_above_12hr_batches": 1.25e-07, "cache_creation_input_token_cost_above_12hr_flex": 1.25e-07, "cache_creation_input_token_cost_above_12hr_priority": 5e-07, "cache_creation_input_token_cost_decisions": 1.25e-07, @@ -37050,7 +35866,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_decisions": 0.0, @@ -37136,7 +35951,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37186,8 +36000,10 @@ "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_12hr": 1e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 1e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05, + "cache_creation_input_token_cost_above_12hr_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_priority": 2e-05, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, @@ -37216,7 +36032,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37268,8 +36083,10 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-06, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -37298,7 +36115,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -37349,8 +36165,10 @@ "cache_creation_input_token_cost": 2.5e-07, "cache_creation_input_token_cost_above_12hr": 5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-07, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-07, "cache_creation_input_token_cost_above_12hr_priority": 1e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, @@ -37379,7 +36197,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_272k_tokens": 1.8e-06, @@ -37437,7 +36254,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, @@ -37476,7 +36292,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, @@ -37515,7 +36330,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 7.5e-05, "output_cost_per_token_above_272k_tokens": 0.0001125, @@ -37553,7 +36367,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37592,7 +36405,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, @@ -37626,7 +36438,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "reasoning_effort_levels": [ @@ -37672,7 +36483,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -37735,7 +36545,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, @@ -37791,7 +36600,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -37843,7 +36651,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -37902,7 +36709,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -37960,7 +36766,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, @@ -38011,7 +36816,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -38064,7 +36868,6 @@ "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, @@ -38120,7 +36923,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_flex": 2.25e-06, @@ -38175,7 +36977,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "output_cost_per_token_flex": 2.25e-06, @@ -38228,7 +37029,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, @@ -38280,7 +37080,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "output_cost_per_token_flex": 6.25e-07, @@ -38327,7 +37126,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 272000, - "max_tokens": 272000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -38370,7 +37168,6 @@ "litellm_provider": "openai", "max_input_tokens": 400000, "max_output_tokens": 272000, - "max_tokens": 272000, "mode": "responses", "output_cost_per_token": 0.00012, "output_cost_per_token_batches": 6e-05, @@ -38419,7 +37216,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -38465,7 +37261,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -38501,7 +37296,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5-chat-latest", @@ -38534,7 +37328,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.4e-05, "output_cost_per_token_priority": 2.8e-05, @@ -38578,7 +37371,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://developers.openai.com/api/docs/models/gpt-5.3-chat-latest", @@ -38615,7 +37407,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -38667,7 +37458,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -38717,7 +37507,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -38767,7 +37556,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -38853,7 +37641,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -38888,7 +37675,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -38923,7 +37709,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -38958,7 +37743,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -38995,7 +37779,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -39033,7 +37816,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -39069,7 +37851,6 @@ "litellm_provider": "openai", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, @@ -39095,7 +37876,6 @@ }, "gradient_ai/alibaba-qwen3-32b": { "litellm_provider": "gradient_ai", - "max_tokens": 40960, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -39110,7 +37890,6 @@ "gradient_ai/anthropic-claude-3-opus": { "input_cost_per_token": 1.5e-05, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 7.5e-05, "supported_endpoints": [ @@ -39126,7 +37905,6 @@ "gradient_ai/anthropic-claude-3.5-haiku": { "input_cost_per_token": 8e-07, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 4e-06, "supported_endpoints": [ @@ -39142,7 +37920,6 @@ "gradient_ai/anthropic-claude-3.5-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.5e-05, "supported_endpoints": [ @@ -39158,7 +37935,6 @@ "gradient_ai/anthropic-claude-3.7-sonnet": { "input_cost_per_token": 3e-06, "litellm_provider": "gradient_ai", - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 1.5e-05, "supported_endpoints": [ @@ -39174,7 +37950,6 @@ "gradient_ai/deepseek-r1-distill-llama-70b": { "input_cost_per_token": 9.9e-07, "litellm_provider": "gradient_ai", - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 9.9e-07, "supported_endpoints": [ @@ -39190,7 +37965,6 @@ "gradient_ai/llama3-8b-instruct": { "input_cost_per_token": 2e-07, "litellm_provider": "gradient_ai", - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 2e-07, "supported_endpoints": [ @@ -39206,7 +37980,6 @@ "gradient_ai/llama3.3-70b-instruct": { "input_cost_per_token": 6.5e-07, "litellm_provider": "gradient_ai", - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 6.5e-07, "supported_endpoints": [ @@ -39222,7 +37995,6 @@ "gradient_ai/mistral-nemo-instruct-2407": { "input_cost_per_token": 3e-07, "litellm_provider": "gradient_ai", - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 3e-07, "supported_endpoints": [ @@ -39237,7 +38009,6 @@ }, "gradient_ai/openai-gpt-4o": { "litellm_provider": "gradient_ai", - "max_tokens": 16384, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -39251,7 +38022,6 @@ }, "gradient_ai/openai-gpt-4o-mini": { "litellm_provider": "gradient_ai", - "max_tokens": 16384, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -39266,7 +38036,6 @@ "gradient_ai/openai-o3": { "input_cost_per_token": 2e-06, "litellm_provider": "gradient_ai", - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "supported_endpoints": [ @@ -39282,7 +38051,6 @@ "gradient_ai/openai-o3-mini": { "input_cost_per_token": 1.1e-06, "litellm_provider": "gradient_ai", - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supported_endpoints": [ @@ -39298,7 +38066,6 @@ "lemonade/Qwen3-Coder-30B-A3B-Instruct-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", @@ -39310,7 +38077,6 @@ "lemonade/gpt-oss-20b-mxfp4-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "chat", @@ -39322,7 +38088,6 @@ "lemonade/gpt-oss-120b-mxfp-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 131072, "max_output_tokens": 32768, "mode": "chat", @@ -39334,7 +38099,6 @@ "lemonade/Gemma-3-4b-it-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "mode": "chat", @@ -39346,7 +38110,6 @@ "lemonade/Qwen3-4B-Instruct-2507-GGUF": { "input_cost_per_token": 0, "litellm_provider": "lemonade", - "max_tokens": 32768, "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", @@ -39360,7 +38123,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.4e-07, "supports_function_calling": true, @@ -39372,7 +38134,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.4e-07, "supports_function_calling": true, @@ -39386,7 +38147,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 1000000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.25e-05, "supports_function_calling": true, @@ -39400,7 +38160,6 @@ "litellm_provider": "amazon_nova", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, "supports_function_calling": true, @@ -39414,7 +38173,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://console.groq.com/docs/model/llama-guard-3-8b" @@ -39424,7 +38183,6 @@ "litellm_provider": "groq", "max_input_tokens": 512, "max_output_tokens": 512, - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 3e-08, "source": "https://console.groq.com/docs/models" @@ -39434,7 +38192,6 @@ "litellm_provider": "groq", "max_input_tokens": 512, "max_output_tokens": 512, - "max_tokens": 512, "mode": "chat", "output_cost_per_token": 4e-08, "source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m" @@ -39445,7 +38202,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "search_context_cost_per_query": { @@ -39466,7 +38222,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "search_context_cost_per_query": { @@ -39487,7 +38242,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "search_context_cost_per_query": { @@ -39506,7 +38260,6 @@ "litellm_provider": "groq", "max_input_tokens": 4000, "max_output_tokens": 50000, - "max_tokens": 50000, "mode": "audio_speech", "source": "https://console.groq.com/docs/model/canopylabs/orpheus-v1-english" }, @@ -39515,7 +38268,6 @@ "litellm_provider": "groq", "max_input_tokens": 4000, "max_output_tokens": 50000, - "max_tokens": 50000, "mode": "audio_speech", "source": "https://console.groq.com/docs/models" }, @@ -39533,7 +38285,6 @@ }, "heroku/claude-3-5-haiku": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39543,7 +38294,6 @@ }, "heroku/claude-3-5-sonnet-latest": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39553,7 +38303,6 @@ }, "heroku/claude-3-7-sonnet": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39563,7 +38312,6 @@ }, "heroku/claude-4-sonnet": { "litellm_provider": "heroku", - "max_tokens": 8192, "mode": "chat", "supports_function_calling": true, "supports_system_messages": true, @@ -39609,7 +38357,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39622,7 +38369,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -39635,7 +38381,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39648,7 +38393,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39661,7 +38405,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -39674,7 +38417,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "supports_function_calling": true, @@ -39687,7 +38429,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_function_calling": true, @@ -39700,7 +38441,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -39713,7 +38453,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "supports_function_calling": true, @@ -39726,7 +38465,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39739,7 +38477,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39752,7 +38489,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39765,7 +38501,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39778,7 +38513,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39791,7 +38525,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -39804,7 +38537,6 @@ "litellm_provider": "hyperbolic", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-06, "supports_function_calling": true, @@ -39817,7 +38549,6 @@ "litellm_provider": "ai21", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 3e-06 }, @@ -39826,7 +38557,6 @@ "litellm_provider": "ai21", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 1e-05 }, @@ -39835,7 +38565,6 @@ "litellm_provider": "ai21", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 1.5e-05 }, @@ -39844,7 +38573,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39854,7 +38582,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39864,7 +38591,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39874,7 +38600,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39884,7 +38609,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39894,7 +38618,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39904,7 +38627,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -39914,7 +38636,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39924,7 +38645,6 @@ "litellm_provider": "ai21", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -39934,7 +38654,6 @@ "litellm_provider": "jina_ai", "max_input_tokens": 1024, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://api.jina.ai/v1/models" @@ -39954,7 +38673,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -39991,7 +38709,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -40019,7 +38736,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 7e-06, "supports_function_calling": false, @@ -40032,7 +38748,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -40045,7 +38760,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -40059,7 +38773,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -40072,7 +38785,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-06, "supports_function_calling": false, @@ -40085,7 +38797,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -40098,7 +38809,6 @@ "litellm_provider": "crusoe", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -40112,7 +38822,6 @@ "litellm_provider": "inception", "max_input_tokens": 128000, "max_output_tokens": 50000, - "max_tokens": 50000, "mode": "chat", "output_cost_per_token": 7.5e-07, "supports_function_calling": true, @@ -40127,7 +38836,6 @@ "litellm_provider": "inception", "max_input_tokens": 260000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.5e-07, "source": "https://docs.inceptionlabs.ai/get-started/models", @@ -40142,7 +38850,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40163,7 +38871,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40184,7 +38892,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40205,7 +38913,7 @@ "input_cost_per_token": 1.72e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40226,7 +38934,7 @@ "input_cost_per_token": 1.72e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40247,7 +38955,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://inferenceapis.com/pricing", @@ -40268,7 +38976,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40289,7 +38997,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://inferenceapis.com/pricing", @@ -40310,7 +39018,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40331,7 +39039,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40352,7 +39060,7 @@ "input_cost_per_token": 1.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.8e-07, "source": "https://inferenceapis.com/pricing", @@ -40373,7 +39081,7 @@ "input_cost_per_token": 1.72e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40394,7 +39102,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://inferenceapis.com/pricing", @@ -40415,7 +39123,7 @@ "input_cost_per_token": 1.82e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 5.72e-06, "source": "https://inferenceapis.com/pricing", @@ -40436,7 +39144,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://inferenceapis.com/pricing", @@ -40456,7 +39164,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://inferenceapis.com/pricing", @@ -40475,7 +39183,7 @@ "input_cost_per_token": 4e-08, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://inferenceapis.com/pricing", @@ -40495,7 +39203,7 @@ "input_cost_per_token": 8.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.42e-06, "source": "https://inferenceapis.com/pricing", @@ -40515,7 +39223,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40534,7 +39242,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40553,7 +39261,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40572,7 +39280,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40591,7 +39299,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40610,7 +39318,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40630,7 +39338,7 @@ "input_cost_per_token": 3.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://inferenceapis.com/pricing", @@ -40651,7 +39359,7 @@ "input_cost_per_token": 3.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://inferenceapis.com/pricing", @@ -40672,7 +39380,7 @@ "input_cost_per_token": 8.9e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.42e-06, "source": "https://inferenceapis.com/pricing", @@ -40692,7 +39400,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://inferenceapis.com/pricing", @@ -40711,7 +39419,7 @@ "input_cost_per_token": 4e-08, "litellm_provider": "inferenceapis", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://inferenceapis.com/pricing", @@ -40730,7 +39438,7 @@ "input_cost_per_token": 2.6e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40750,7 +39458,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40770,7 +39478,7 @@ "input_cost_per_token": 2.6e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40789,7 +39497,7 @@ "input_cost_per_token": 2.6e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://inferenceapis.com/pricing", @@ -40809,7 +39517,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inferenceapis.com/pricing", @@ -40830,7 +39538,7 @@ "input_cost_per_token": 1.82e-06, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 5.72e-06, "source": "https://inferenceapis.com/pricing", @@ -40851,7 +39559,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "inferenceapis", "max_input_tokens": 1048575, - "max_tokens": 1048575, + "max_output_tokens": 1048575, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://inferenceapis.com/pricing", @@ -40873,7 +39581,6 @@ "litellm_provider": "text-completion-inception", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 7.5e-07 }, @@ -40882,7 +39589,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -40896,7 +39602,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -40910,7 +39615,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -40924,7 +39628,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -40937,7 +39640,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -40950,7 +39652,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -40963,7 +39664,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -40976,7 +39676,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "supports_function_calling": true, @@ -40989,7 +39688,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -41002,7 +39700,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41015,7 +39712,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 16384, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41028,7 +39724,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -41041,7 +39736,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -41054,7 +39748,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -41067,7 +39760,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -41080,7 +39772,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-08, "supports_function_calling": true, @@ -41094,7 +39785,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-08, "supports_function_calling": true, @@ -41107,7 +39797,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -41120,7 +39809,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41133,7 +39821,6 @@ "litellm_provider": "lambda_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "supports_function_calling": true, @@ -41178,14 +39865,12 @@ "max-x-max/50-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.036 }, "max-x-max/max-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.072 }, @@ -41281,7 +39966,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_character": 1.5e-05, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -41292,7 +39976,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_character": 1e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", @@ -41303,7 +39986,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -41313,7 +39995,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.56e-06, "source": "https://aws.amazon.com/bedrock/pricing/" @@ -41324,7 +40005,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -41338,7 +40018,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -41353,7 +40032,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.2e-07, "output_cost_per_token_batches": 1.1e-07, @@ -41367,7 +40045,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.6e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -41381,7 +40058,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", @@ -41394,7 +40070,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", @@ -41407,7 +40082,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -41422,7 +40096,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -41436,7 +40109,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md" @@ -41447,7 +40119,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md" @@ -41459,7 +40130,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9.7e-07, "output_cost_per_token_batches": 4.85e-07, @@ -41482,7 +40152,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 10000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_batches": 3.3e-07, @@ -41504,7 +40173,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.25e-06, "search_context_cost_per_query": { @@ -41545,7 +40213,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.25e-06, "search_context_cost_per_query": { @@ -41586,7 +40253,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "search_context_cost_per_query": { @@ -41627,7 +40293,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.25e-06, "search_context_cost_per_query": { @@ -41668,7 +40333,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "search_context_cost_per_query": { @@ -41725,7 +40389,6 @@ "litellm_provider": "meta", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-05, "search_context_cost_per_query": { @@ -41766,7 +40429,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 128000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41782,7 +40444,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 128000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41798,7 +40459,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 1000000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41815,7 +40475,6 @@ "litellm_provider": "meta_llama", "max_input_tokens": 10000000, "max_output_tokens": 4028, - "max_tokens": 4028, "mode": "chat", "source": "https://ai.developer.meta.com/docs/models", "supported_modalities": [ @@ -41837,7 +40496,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -41860,7 +40518,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -41884,7 +40541,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 6e-07, "output_cost_per_token_priority": 2.1e-06, @@ -42082,7 +40738,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_batches": 1e-06, @@ -42104,7 +40759,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 40000, - "max_tokens": 40000, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, @@ -42126,7 +40780,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_batches": 1e-07, @@ -42148,7 +40801,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-07, "output_cost_per_token_batches": 5e-08, @@ -42170,7 +40822,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-07, "output_cost_per_token_batches": 7e-08, @@ -42190,7 +40841,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-7b-instruct.md", @@ -42201,7 +40851,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mistral-large.md", @@ -42212,7 +40861,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -42227,7 +40875,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_batches": 7.5e-07, @@ -42247,7 +40894,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -42260,7 +40906,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-mixtral-8x7b-instruct.md", @@ -42275,7 +40920,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-08, "output_cost_per_token_batches": 2e-08, @@ -42295,7 +40939,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, "output_cost_per_token_batches": 1.5e-07, @@ -42312,7 +40955,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -42327,7 +40969,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -42342,7 +40983,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://mistral.ai/technology/", @@ -42355,7 +40995,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://docs.mistral.ai/models/devstral-small-2-25-12", @@ -42370,7 +41009,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://mistral.ai/news/devstral-2-vibe-cli", @@ -42385,7 +41023,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://mistral.ai/news/devstral-2-vibe-cli", @@ -42399,7 +41036,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12", @@ -42415,7 +41051,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://docs.mistral.ai/models/ministral-3-14b-25-12", @@ -42431,7 +41066,6 @@ "litellm_provider": "mistral", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12", @@ -42447,7 +41081,6 @@ "litellm_provider": "mistral", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.mistral.ai/models/ministral-3-3b-25-12", @@ -42462,7 +41095,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "source": "https://docs.mistral.ai/models/mistral-embed-23-12" }, @@ -42472,7 +41104,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -42653,7 +41284,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://docs.mistral.ai/models/voxtral-small-25-07", @@ -42668,7 +41298,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://docs.mistral.ai/models/voxtral-small-25-07", @@ -42684,7 +41313,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42710,7 +41338,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42732,7 +41359,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "reasoning_effort_levels": [ @@ -42755,7 +41381,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42777,7 +41402,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42799,7 +41423,6 @@ "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "reasoning_effort_levels": [ @@ -42878,7 +41501,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", @@ -42895,7 +41517,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://docs.mistral.ai/models/model-cards/mistral-small-4-0-26-03", @@ -42910,7 +41531,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding" }, "mistral/codestral-embed": { @@ -42918,7 +41538,6 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding" }, "mistral/codestral-embed-2505": { @@ -42926,7 +41545,6 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "mistral", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding" }, "mistral/mistral-large-4": { @@ -42934,7 +41552,7 @@ "input_cost_per_token": 6.8e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.09e-06, "reasoning_effort_levels": [ @@ -42955,7 +41573,7 @@ "input_cost_per_token": 6.8e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.09e-06, "reasoning_effort_levels": [ @@ -42977,7 +41595,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", @@ -42993,7 +41610,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", @@ -43009,7 +41625,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://docs.mistral.ai/models/mistral-large-3-25-12", @@ -43025,7 +41640,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43046,7 +41660,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43067,7 +41680,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43088,7 +41700,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -43109,7 +41720,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-07, "supports_assistant_prefill": true, @@ -43123,7 +41733,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "reasoning_effort_levels": [ @@ -43144,7 +41753,6 @@ "litellm_provider": "mistral", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://mistral.ai/pricing", @@ -43160,7 +41768,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://mistral.ai/pricing", @@ -43176,7 +41783,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://mistral.ai/pricing", @@ -43192,7 +41798,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://mistral.ai/pricing", @@ -43208,7 +41813,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://mistral.ai/pricing", @@ -43224,7 +41828,6 @@ "litellm_provider": "mistral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_assistant_prefill": true, @@ -43237,7 +41840,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://mistral.ai/technology/", @@ -43251,7 +41853,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-06, "supports_assistant_prefill": true, @@ -43268,7 +41869,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, @@ -43290,7 +41890,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, @@ -43310,7 +41909,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://platform.kimi.ai/docs/pricing/chat-k27-code", @@ -43328,7 +41926,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", @@ -43345,7 +41942,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://platform.kimi.ai/docs/pricing/chat-k26", @@ -43362,7 +41958,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ @@ -43383,7 +41978,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43396,7 +41990,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43410,7 +42003,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43423,7 +42015,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43437,7 +42028,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43450,7 +42040,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43464,7 +42053,6 @@ "litellm_provider": "moonshot", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", @@ -43477,7 +42065,6 @@ "litellm_provider": "morph", "max_input_tokens": 16000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": false, @@ -43491,7 +42078,6 @@ "litellm_provider": "morph", "max_input_tokens": 16000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.9e-06, "supports_function_calling": false, @@ -43509,7 +42095,6 @@ "input_cost_per_video_per_second_above_8s_interval": 0.001, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -43533,7 +42118,6 @@ "input_cost_per_video_per_second_above_8s_interval": 0.001, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -43693,7 +42277,6 @@ ] }, "nebius/deepseek-ai/DeepSeek-R1": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 8e-07, @@ -43705,7 +42288,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-0528": { - "max_tokens": 164000, "max_input_tokens": 164000, "max_output_tokens": 164000, "input_cost_per_token": 8e-07, @@ -43717,7 +42299,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2.5e-07, @@ -43728,7 +42309,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 5e-07, @@ -43739,7 +42319,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V3-0324": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 5e-07, @@ -43750,7 +42329,6 @@ "source": "https://nebius.com/prices" }, "nebius/google/gemma-3-27b-it": { - "max_tokens": 110000, "max_input_tokens": 110000, "max_output_tokens": 110000, "input_cost_per_token": 1e-07, @@ -43762,7 +42340,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, @@ -43773,7 +42350,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Llama-Guard-3-8B": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-08, @@ -43783,7 +42359,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-8B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-08, @@ -43794,7 +42369,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.3e-07, @@ -43805,7 +42379,6 @@ "source": "https://nebius.com/prices" }, "nebius/meta-llama/Meta-Llama-3.1-405B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1e-06, @@ -43816,7 +42389,6 @@ "source": "https://nebius.com/prices" }, "nebius/mistralai/Mistral-Nemo-Instruct-2407": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 4e-08, @@ -43827,7 +42399,6 @@ "source": "https://nebius.com/prices" }, "nebius/NousResearch/Hermes-3-Llama-3.1-405B": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1e-06, @@ -43838,7 +42409,6 @@ "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 6e-07, @@ -43849,7 +42419,6 @@ "source": "https://nebius.com/prices" }, "nebius/nvidia/Llama-3.3-Nemotron-Super-49B-v1": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -43860,7 +42429,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-235B-A22B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2e-07, @@ -43871,7 +42439,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-32B": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-07, @@ -43882,7 +42449,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-32B" }, "nebius/Qwen/Qwen3-30B-A3B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -43893,7 +42459,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-14B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 8e-08, @@ -43904,7 +42469,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-4B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 8e-08, @@ -43915,7 +42479,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/QwQ-32B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.5e-07, @@ -43927,7 +42490,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-72B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.3e-07, @@ -43938,7 +42500,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-32B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 6e-08, @@ -43949,7 +42510,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-Coder-7B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-08, @@ -43960,7 +42520,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2.5-VL-72B-Instruct": { - "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, "input_cost_per_token": 2.5e-07, @@ -43972,7 +42531,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/Qwen%2FQwen2.5-VL-72B-Instruct" }, "nebius/Qwen/Qwen2-VL-72B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, @@ -43984,7 +42542,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen2-VL-7B-Instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-08, @@ -43995,7 +42552,6 @@ "source": "https://nebius.com/prices" }, "nebius/deepseek-ai/DeepSeek-V4-Flash": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.4e-07, @@ -44007,7 +42563,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash" }, "nebius/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_tokens": 1024000, "max_input_tokens": 1024000, "max_output_tokens": 1024000, "input_cost_per_token": 1.4e-07, @@ -44019,7 +42574,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731" }, "nebius/deepseek-ai/DeepSeek-V4-Pro": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.75e-06, @@ -44045,7 +42599,6 @@ "litellm_provider": "nebius", "max_input_tokens": 1048000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://tokenfactory.nebius.com/endpoints?modals=endpoint-details&model-id=deepseek-ai/DeepSeek-V4.1-Flash", @@ -44054,7 +42607,6 @@ "supports_vision": true }, "nebius/MiniMaxAI/MiniMax-M2.5": { - "max_tokens": 196608, "max_input_tokens": 196608, "max_output_tokens": 196608, "input_cost_per_token": 3e-07, @@ -44066,7 +42618,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M2.5" }, "nebius/MiniMaxAI/MiniMax-M3": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 3e-07, @@ -44079,7 +42630,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3" }, "nebius/moonshotai/Kimi-K2.6": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9.5e-07, @@ -44092,7 +42642,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6" }, "nebius/moonshotai/Kimi-K2.7-Code": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9.5e-07, @@ -44105,7 +42654,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" }, "nebius/moonshotai/Kimi-K3": { - "max_tokens": 1024000, "max_input_tokens": 1024000, "max_output_tokens": 1024000, "input_cost_per_token": 3e-06, @@ -44118,7 +42666,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3" }, "nebius/NousResearch/Hermes-4-405B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-06, @@ -44130,7 +42677,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B" }, "nebius/NousResearch/Hermes-4-70B": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.3e-07, @@ -44142,7 +42688,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-70B" }, "nebius/nvidia/Cosmos3-Super-Reasoner": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-07, @@ -44155,7 +42700,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/nvidia%2FCosmos3-Super-Reasoner" }, "nebius/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6e-07, @@ -44166,7 +42710,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FLlama-3_1-Nemotron-Ultra-253B-v1" }, "nebius/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 6e-08, @@ -44178,7 +42721,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B" }, "nebius/nvidia/Nemotron-3-Nano-Omni": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 6e-08, @@ -44191,7 +42733,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Nano-Omni" }, "nebius/nvidia/nemotron-3-super-120b-a12b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 3e-07, @@ -44203,7 +42744,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b" }, "nebius/nvidia/Nemotron-3-Ultra-550b-a55b": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1e-06, @@ -44215,7 +42755,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b" }, "nebius/nvidia/Nemotron-3_5-Lightning": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 6e-08, @@ -44227,7 +42766,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning" }, "nebius/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.5e-07, @@ -44239,7 +42777,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b" }, "nebius/openbmb/MiniCPM-V-4_5": { - "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, "input_cost_per_token": 6.58e-07, @@ -44250,7 +42787,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5" }, "nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2e-07, @@ -44261,7 +42797,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507" }, "nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-07, @@ -44272,7 +42807,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507" }, "nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.5e-07, @@ -44284,7 +42818,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-Next-80B-A3B-Thinking" }, "nebius/Qwen/Qwen3.5-397B-A17B": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 6e-07, @@ -44306,7 +42839,6 @@ "supports_reasoning": true }, "nebius/zai-org/GLM-5.1": { - "max_tokens": 202752, "max_input_tokens": 202752, "max_output_tokens": 202752, "input_cost_per_token": 1.4e-06, @@ -44318,7 +42850,6 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1" }, "nebius/zai-org/GLM-5.2": { - "max_tokens": 1048576, "max_input_tokens": 1048576, "max_output_tokens": 1048576, "input_cost_per_token": 1.4e-06, @@ -44333,7 +42864,7 @@ "input_cost_per_token": 1.4e-06, "litellm_provider": "nebius", "max_input_tokens": 1024000, - "max_tokens": 1024000, + "max_output_tokens": 1024000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3", @@ -44341,7 +42872,6 @@ "supports_reasoning": true }, "nebius/zai-org/GLM-5.3-Flash": { - "max_tokens": 1024000, "max_input_tokens": 1024000, "max_output_tokens": 1024000, "input_cost_per_token": 1.5e-07, @@ -44354,7 +42884,6 @@ "supports_vision": true }, "nebius/BAAI/bge-en-icl": { - "max_tokens": 32768, "max_input_tokens": 32768, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44363,7 +42892,6 @@ "source": "https://nebius.com/prices" }, "nebius/BAAI/bge-multilingual-gemma2": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44372,7 +42900,6 @@ "source": "https://nebius.com/prices" }, "nebius/intfloat/e5-mistral-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44381,7 +42908,6 @@ "source": "https://nebius.com/prices" }, "nebius/Qwen/Qwen3-Embedding-8B": { - "max_tokens": 40960, "max_input_tokens": 40960, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -44398,7 +42924,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -44420,7 +42945,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.3e-07, "output_cost_per_token_batches": 1.2e-07, @@ -44441,7 +42965,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07, @@ -44464,7 +42987,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.5e-07, "output_cost_per_token_batches": 3.25e-07, @@ -44487,7 +43009,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -44510,7 +43031,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_batches": 3e-05, @@ -44532,7 +43052,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 0.0006, "output_cost_per_token_batches": 0.0003, @@ -44566,7 +43085,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 0.0006, "output_cost_per_token_batches": 0.0003, @@ -44604,7 +43122,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -44645,7 +43162,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, "source": "https://developers.openai.com/api/docs/models/o3-deep-research", @@ -44678,7 +43194,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 4e-05, "source": "https://developers.openai.com/api/docs/models/o3-deep-research", @@ -44715,7 +43230,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "output_cost_per_token_batches": 4e-06, @@ -44758,7 +43272,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -44779,7 +43292,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -44798,7 +43310,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -44836,7 +43347,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-05, "output_cost_per_token_batches": 4e-05, @@ -44879,7 +43389,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -44907,7 +43416,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-06, "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research", @@ -44940,7 +43448,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "responses", "output_cost_per_token": 8e-06, "source": "https://developers.openai.com/api/docs/models/o4-mini-deep-research", @@ -44977,7 +43484,6 @@ "litellm_provider": "openai", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_batches": 2.2e-06, @@ -45004,7 +43510,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45017,7 +43522,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45030,7 +43534,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.068e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45043,7 +43546,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45057,7 +43559,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45070,7 +43571,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45084,7 +43584,6 @@ "litellm_provider": "oci", "max_input_tokens": 10485760, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45097,7 +43596,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45110,7 +43608,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45123,7 +43620,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45136,7 +43632,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45149,7 +43644,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45162,7 +43656,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45175,7 +43668,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45188,7 +43680,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45201,7 +43692,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45221,7 +43711,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45240,7 +43729,6 @@ "litellm_provider": "oci", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45260,7 +43748,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45274,7 +43761,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/enterprise-ai/cost-estimator/", @@ -45296,7 +43782,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45308,7 +43793,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45321,7 +43805,6 @@ "litellm_provider": "oci", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 9e-08, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45333,7 +43816,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45345,7 +43827,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45357,7 +43838,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45370,7 +43850,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45382,7 +43861,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45394,7 +43872,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45406,7 +43883,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [], @@ -45419,7 +43895,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45434,7 +43909,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45450,7 +43924,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45467,7 +43940,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45483,7 +43955,6 @@ "litellm_provider": "oci", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -45497,7 +43968,6 @@ "litellm_provider": "oci", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45509,7 +43979,6 @@ "litellm_provider": "oci", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45524,7 +43993,6 @@ "litellm_provider": "oci", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45539,7 +44007,6 @@ "litellm_provider": "oci", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45554,7 +44021,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45567,7 +44033,6 @@ "litellm_provider": "oci", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", @@ -45579,7 +44044,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -45589,7 +44053,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45599,7 +44062,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -45609,7 +44071,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45619,7 +44080,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -45630,7 +44090,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45641,7 +44100,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "oci", "max_input_tokens": 512, - "max_tokens": 512, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 384, @@ -45652,7 +44110,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "oci", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -45664,7 +44121,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": false @@ -45674,7 +44130,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45683,7 +44138,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45692,7 +44146,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45702,7 +44155,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45712,7 +44164,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45722,7 +44173,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45732,7 +44182,6 @@ "litellm_provider": "ollama", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45742,7 +44191,6 @@ "litellm_provider": "ollama", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45752,7 +44200,6 @@ "litellm_provider": "ollama", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45762,7 +44209,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45772,7 +44218,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45781,7 +44226,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45790,7 +44234,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45799,7 +44242,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45808,7 +44250,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45817,7 +44258,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45826,7 +44266,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45836,7 +44275,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45845,7 +44283,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -45854,7 +44291,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "completion", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45864,7 +44300,6 @@ "litellm_provider": "ollama", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45874,7 +44309,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45884,7 +44318,6 @@ "litellm_provider": "ollama", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45894,7 +44327,6 @@ "litellm_provider": "ollama", "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45904,7 +44336,6 @@ "litellm_provider": "ollama", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45914,7 +44345,6 @@ "litellm_provider": "ollama", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45923,7 +44353,6 @@ "litellm_provider": "ollama", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 0.0, "supports_function_calling": true @@ -45933,7 +44362,6 @@ "litellm_provider": "ollama", "max_input_tokens": 2048, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -45942,7 +44370,6 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -45951,7 +44378,6 @@ "litellm_provider": "openai", "max_input_tokens": 32768, "max_output_tokens": 0, - "max_tokens": 0, "mode": "moderation", "output_cost_per_token": 0.0 }, @@ -45969,7 +44395,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -45995,7 +44420,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07, "output_cost_per_token_batches": 1.5e-07, @@ -46015,7 +44439,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46037,7 +44460,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_batches": 1e-07, @@ -46056,7 +44478,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46070,7 +44491,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46083,7 +44503,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46096,7 +44515,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 6.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46109,7 +44527,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 127000, "max_output_tokens": 127000, - "max_tokens": 127000, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46122,7 +44539,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 118000, "max_output_tokens": 118000, - "max_tokens": 118000, "mode": "chat", "output_cost_per_token": 1.3e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46135,7 +44551,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46149,7 +44564,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.3e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46162,7 +44576,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 8.7e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46175,7 +44588,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 9.1e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46189,7 +44601,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.3e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46203,7 +44614,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46217,7 +44627,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 131000, "max_output_tokens": 131000, - "max_tokens": 131000, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46231,7 +44640,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2.9e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46245,7 +44653,6 @@ "litellm_provider": "ovhcloud", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.9e-07, "source": "https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog/", @@ -46278,7 +44685,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.4e-06 }, @@ -46287,7 +44693,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46296,7 +44701,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46305,7 +44709,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -46314,7 +44717,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07 }, @@ -46323,7 +44725,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46332,7 +44733,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46341,7 +44741,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46351,7 +44750,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-06 }, @@ -46360,7 +44758,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46370,7 +44767,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46426,7 +44822,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-06, "search_context_cost_per_query": { @@ -46441,7 +44837,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_reasoning_token": 3e-06, "output_cost_per_token": 8e-06, @@ -46458,7 +44854,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.8e-06 }, @@ -46468,7 +44863,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 12000, "max_output_tokens": 12000, - "max_tokens": 12000, "mode": "chat", "output_cost_per_token": 1.8e-06 }, @@ -46477,7 +44871,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 200000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -46491,7 +44884,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-06, "search_context_cost_per_query": { @@ -46506,7 +44899,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "perplexity", "max_input_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 8e-06, "search_context_cost_per_query": { @@ -46522,7 +44915,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46532,7 +44924,6 @@ "litellm_provider": "perplexity", "max_input_tokens": 12000, "max_output_tokens": 12000, - "max_tokens": 12000, "mode": "chat", "output_cost_per_token": 2.8e-07 }, @@ -46541,7 +44932,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46553,7 +44943,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46565,7 +44954,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46577,7 +44965,6 @@ "litellm_provider": "publicai", "max_input_tokens": 16384, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46589,7 +44976,6 @@ "litellm_provider": "publicai", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46601,7 +44987,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46841,7 +45226,6 @@ "input_cost_per_token": 4e-09, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -46851,7 +45235,6 @@ "input_cost_per_token": 3e-08, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2560, @@ -46862,7 +45245,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46874,7 +45256,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46887,7 +45268,6 @@ "litellm_provider": "publicai", "max_input_tokens": 32768, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", @@ -46901,7 +45281,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.8e-06, "supports_function_calling": true, @@ -46916,7 +45295,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 8.8e-07, "output_cost_per_token_flex": 4.4e-07, @@ -46936,7 +45314,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46955,7 +45332,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, @@ -46972,7 +45348,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -46988,7 +45363,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47004,7 +45378,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47020,7 +45393,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.236e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47036,7 +45408,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47052,7 +45423,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47068,7 +45438,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", @@ -47086,7 +45455,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.66e-06, "output_cost_per_token_batches": 1.33e-06, @@ -47107,7 +45475,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_batches": 6e-07, @@ -47162,7 +45529,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, "supports_tool_choice": true @@ -47172,7 +45538,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 5e-07, "supports_tool_choice": true @@ -47182,7 +45547,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47192,7 +45556,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47202,7 +45565,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47212,7 +45574,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47222,7 +45583,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47232,7 +45592,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.75e-06, "supports_tool_choice": true @@ -47242,7 +45601,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8086, "max_output_tokens": 8086, - "max_tokens": 8086, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47252,7 +45610,6 @@ "litellm_provider": "replicate", "max_input_tokens": 8086, "max_output_tokens": 8086, - "max_tokens": 8086, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47262,7 +45619,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47272,7 +45628,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.5e-07, "supports_tool_choice": true @@ -47282,7 +45637,6 @@ "litellm_provider": "replicate", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-06, "supports_tool_choice": true @@ -47411,7 +45765,6 @@ "mode": "chat", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_system_messages": true }, @@ -47566,7 +45919,6 @@ "mode": "chat", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true @@ -47587,7 +45939,6 @@ "mode": "chat", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_reasoning": true, "supports_system_messages": true }, @@ -47597,7 +45948,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -47607,7 +45957,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -47617,7 +45966,6 @@ "litellm_provider": "cohere", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -47627,7 +45975,6 @@ "litellm_provider": "cohere", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://cohere.com/pricing" @@ -47638,7 +45985,6 @@ "litellm_provider": "cohere", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://cohere.com/pricing" @@ -47669,7 +46015,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -47678,7 +46023,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -47687,7 +46031,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -47696,7 +46039,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -47705,7 +46047,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "completion", "output_cost_per_token": 0.0 }, @@ -47714,7 +46055,6 @@ "litellm_provider": "sagemaker", "max_input_tokens": 4096, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -47723,7 +46063,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 196608, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://cloud.sambanova.ai/plans/pricing", @@ -47736,7 +46075,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7e-06, "source": "https://cloud.sambanova.ai/plans/pricing" @@ -47746,7 +46084,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "metadata": { "notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount" }, @@ -47763,7 +46100,6 @@ "litellm_provider": "sambanova", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.sambanova.ai/plans/pricing", @@ -47772,7 +46108,6 @@ "supports_tool_choice": true }, "sambanova/DeepSeek-V3.1": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-06, @@ -47785,7 +46120,6 @@ "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.2e-07, @@ -47798,7 +46132,6 @@ "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/DeepSeek-V3.2": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 3e-06, @@ -47810,7 +46143,6 @@ "source": "https://cloud.sambanova.ai/plans/pricing" }, "sambanova/gemma-4-31B-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3.8e-07, @@ -47826,7 +46158,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 375000, - "max_tokens": 375000, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://scx.ai/models/deepseek-v4-1-flash", @@ -47841,7 +46172,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.08e-06, "source": "https://scx.ai/models/glm-5-3", @@ -47857,7 +46187,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.413e-05, "source": "https://scx.ai/models/kimi-k3", @@ -47873,7 +46202,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.98e-06, "source": "https://scx.ai/pricing", @@ -47890,7 +46218,6 @@ "litellm_provider": "scx-ai", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.99e-06, "source": "https://scx.ai/pricing", @@ -47905,7 +46232,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 200000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -47921,7 +46247,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 1.35e-06, "output_cost_per_token": 5.4e-06, @@ -47932,56 +46257,48 @@ "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-1.5-large": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-1.5-mini": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/jamba-instruct": { "litellm_provider": "snowflake", "max_input_tokens": 256000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama2-70b-chat": { "litellm_provider": "snowflake", "max_input_tokens": 4096, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3-70b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3-8b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.1-405b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 1.2e-06, "output_cost_per_token": 1.2e-06, @@ -47992,7 +46309,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 7.2e-07, "output_cost_per_token": 7.2e-07, @@ -48003,7 +46319,6 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 2.4e-07, "output_cost_per_token": 2.4e-07, @@ -48013,18 +46328,15 @@ "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.2-3b": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/llama3.3-70b": { - "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 7.2e-07, @@ -48038,21 +46350,18 @@ "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/mistral-large": { "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/mistral-large2": { "litellm_provider": "snowflake", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, @@ -48064,39 +46373,33 @@ "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/reka-core": { "litellm_provider": "snowflake", "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/reka-flash": { "litellm_provider": "snowflake", "max_input_tokens": 100000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-arctic": { "litellm_provider": "snowflake", "max_input_tokens": 4096, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-llama-3.1-405b": { "litellm_provider": "snowflake", "max_input_tokens": 8000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat" }, "snowflake/snowflake-llama-3.3-70b": { - "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 7.2e-07, @@ -48293,21 +46596,18 @@ "stability.sd3-5-large-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.08 }, "stability.sd3-large-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.08 }, "stability.stable-image-core-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.04 }, @@ -48483,21 +46783,18 @@ "stability.stable-image-core-v1:1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.04 }, "stability.stable-image-ultra-v1:0": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.14 }, "stability.stable-image-ultra-v1:1": { "litellm_provider": "bedrock", "max_input_tokens": 77, - "max_tokens": 77, "mode": "image_generation", "output_cost_per_image": 0.14 }, @@ -48531,7 +46828,6 @@ "litellm_provider": "text-completion-codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "completion", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/" @@ -48541,7 +46837,6 @@ "litellm_provider": "text-completion-codestral", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "completion", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/capabilities/code_generation/" @@ -48552,7 +46847,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48564,7 +46858,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48575,7 +46868,6 @@ "input_cost_per_token_batches": 6.5e-08, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_cost_per_token_batches": 0.0, @@ -48587,7 +46879,6 @@ "input_cost_per_token_batches": 1e-08, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_cost_per_token_batches": 0.0, @@ -48598,7 +46889,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1536, @@ -48609,7 +46899,6 @@ "input_cost_per_token_batches": 5e-08, "litellm_provider": "openai", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0, "output_cost_per_token_batches": 0.0 @@ -48619,7 +46908,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, @@ -48629,7 +46917,6 @@ "input_cost_per_token": 6.25e-09, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 3072, - "max_tokens": 3072, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48641,7 +46928,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 2048, - "max_tokens": 2048, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 768, @@ -48652,7 +46938,6 @@ "litellm_provider": "vertex_ai-text-models", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "completion", "output_cost_per_token": 2.8e-05, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -48662,7 +46947,6 @@ "litellm_provider": "vertex_ai-text-models", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "completion", "output_cost_per_token": 2.8e-05, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models" @@ -48688,7 +46972,8 @@ "together-ai-8.1b-21b": { "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", - "max_tokens": 1000, + "max_output_tokens": 1000, + "max_input_tokens": 1000, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -48741,7 +47026,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 65536, "max_output_tokens": 8192, - "max_tokens": 8192, "metadata": { "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" }, @@ -48757,7 +47041,7 @@ "input_cost_per_token": 1.04e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.04e-06, "source": "https://api.together.ai/v1/models", @@ -48784,7 +47068,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://api.together.ai/v1/models", @@ -48805,7 +47089,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "metadata": { "successor": "together_ai/zai-org/GLM-5.2" }, @@ -48822,7 +47106,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -48839,7 +47123,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://www.together.ai/pricing", @@ -48849,7 +47133,7 @@ "input_cost_per_token": 0.0, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.together.ai/docs/serverless-models" @@ -48858,7 +47142,7 @@ "input_cost_per_token": 1.7e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://api.together.ai/v1/models", @@ -48873,7 +47157,7 @@ "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://api.together.ai/v1/models", @@ -48884,7 +47168,7 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://www.together.ai/pricing", @@ -48894,7 +47178,7 @@ "input_cost_per_token": 3.2e-07, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.28e-06, "source": "https://api.together.ai/v1/models" @@ -48904,7 +47188,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 1010000, - "max_tokens": 1010000, + "max_output_tokens": 1010000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://api.together.ai/v1/models", @@ -48914,7 +47198,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://api.together.ai/v1/models" @@ -48925,7 +47209,7 @@ "input_cost_per_token": 1.4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://api.together.ai/v1/models", @@ -48940,7 +47224,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -48954,7 +47238,7 @@ "input_cost_per_token": 1.32e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.96e-06, "source": "https://api.together.ai/v1/models", @@ -48969,7 +47253,7 @@ "input_cost_per_token": 3.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models", @@ -48980,7 +47264,7 @@ "input_cost_per_token": 1.25e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.25e-06, "source": "https://docs.together.ai/docs/serverless-models", @@ -48993,7 +47277,7 @@ "input_cost_per_token": 3e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ @@ -49015,7 +47299,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 524288, - "max_tokens": 524288, + "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 4.05e-06, "source": "https://api.together.ai/v1/models", @@ -49031,7 +47315,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 1048575, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.together.ai/v1/models", @@ -49048,7 +47331,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 1048575, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.together.ai/v1/models", @@ -49065,7 +47347,6 @@ "litellm_provider": "together_ai", "max_input_tokens": 1048575, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://api.together.ai/v1/models", @@ -49150,7 +47431,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07, @@ -49169,7 +47449,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 1.4e-07, "output_cost_per_token_batches": 7e-08, @@ -49190,7 +47469,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, - "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, "output_cost_per_token_batches": 1.6e-06, @@ -49211,7 +47489,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_assistant_prefill": true, @@ -49231,7 +47508,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -49259,7 +47535,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -49277,7 +47552,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -49296,7 +47570,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -49314,7 +47587,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -49332,7 +47604,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -49371,7 +47642,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.65e-05, "search_context_cost_per_query": { @@ -49413,7 +47683,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -49445,7 +47714,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, @@ -49485,7 +47753,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, @@ -49524,7 +47791,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, @@ -49562,7 +47828,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -49600,7 +47865,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_adaptive_thinking": true, @@ -49636,7 +47900,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -49654,7 +47917,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", @@ -49670,7 +47932,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", @@ -49685,7 +47946,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -49703,7 +47963,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", @@ -49723,7 +47982,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", @@ -49750,7 +48008,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "supports_function_calling": true, "supports_prompt_caching": false, @@ -49767,7 +48024,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "supports_assistant_prefill": true, @@ -49797,7 +48053,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "search_context_cost_per_query": { @@ -49825,7 +48080,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -49862,7 +48116,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "search_context_cost_per_query": { @@ -49899,7 +48152,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { @@ -49939,7 +48191,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "search_context_cost_per_query": { @@ -49970,7 +48221,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", @@ -49986,7 +48236,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.85e-06, "output_cost_per_token_batches": 9.25e-07, @@ -50007,7 +48256,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 163840, - "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, @@ -50025,7 +48273,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -50039,7 +48286,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -50054,7 +48300,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2.2e-07, "output_cost_per_token_batches": 1.1e-07, @@ -50068,7 +48313,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.6e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -50082,7 +48326,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-1b-instruct.md", @@ -50095,7 +48338,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-2-3b-instruct.md", @@ -50108,7 +48350,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", @@ -50123,7 +48364,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -50138,7 +48378,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9.7e-07, "output_cost_per_token_batches": 4.85e-07, @@ -50161,7 +48400,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 10000000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_batches": 3.3e-07, @@ -50182,7 +48420,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-pixtral-large.md", @@ -50194,7 +48431,6 @@ "litellm_provider": "v0", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -50208,7 +48444,6 @@ "litellm_provider": "v0", "max_input_tokens": 512000, "max_output_tokens": 512000, - "max_tokens": 512000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_function_calling": true, @@ -50222,7 +48457,6 @@ "litellm_provider": "v0", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -50236,7 +48470,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.4e-07 }, @@ -50245,7 +48478,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07 }, @@ -50254,7 +48486,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07 }, @@ -50263,7 +48494,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 40960, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -50274,7 +48504,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 262144, "max_output_tokens": 66536, - "max_tokens": 66536, "mode": "chat", "output_cost_per_token": 1.6e-06, "supports_function_calling": true, @@ -50285,7 +48514,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 300000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.4e-07, "supports_vision": true, @@ -50297,7 +48525,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.4e-07, "supports_function_calling": true, @@ -50308,7 +48535,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 300000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3.2e-06, "supports_vision": true, @@ -50320,7 +48546,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50331,7 +48556,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, "supports_vision": true, @@ -50346,7 +48570,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_vision": true, @@ -50361,7 +48584,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-06, "supports_vision": true, @@ -50376,7 +48598,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -50391,7 +48612,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -50406,7 +48626,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_vision": true, @@ -50421,7 +48640,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -50434,7 +48652,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50452,7 +48669,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50470,7 +48686,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50489,7 +48704,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -50509,7 +48723,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -50529,7 +48742,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -50549,7 +48761,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": true, @@ -50572,7 +48783,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_assistant_prefill": false, @@ -50593,7 +48803,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50613,7 +48822,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -50630,7 +48838,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -50642,7 +48849,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -50653,7 +48859,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -50664,7 +48869,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50673,7 +48877,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.19e-06, "supports_tool_choice": true @@ -50683,7 +48886,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 9.9e-07, "supports_function_calling": true, @@ -50695,7 +48897,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 9e-07, "supports_tool_choice": true @@ -50705,7 +48906,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.5e-06, "supports_vision": true, @@ -50724,7 +48924,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "supports_vision": true, @@ -50745,7 +48944,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -50754,7 +48952,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "supports_vision": true, @@ -50766,7 +48963,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -50775,7 +48971,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -50784,7 +48979,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -50793,7 +48987,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.9e-07, "supports_tool_choice": true @@ -50803,7 +48996,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 8e-08, "supports_tool_choice": true @@ -50813,7 +49005,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_tool_choice": true @@ -50823,7 +49014,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-08, "supports_function_calling": true, @@ -50834,7 +49024,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.6e-07, "supports_vision": true, @@ -50846,7 +49035,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1e-07 }, @@ -50855,7 +49043,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_function_calling": true, @@ -50866,7 +49053,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_vision": true, @@ -50878,7 +49064,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "supports_function_calling": true, @@ -50889,7 +49074,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, "supports_tool_choice": true @@ -50899,7 +49083,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, "supports_vision": true, @@ -50911,7 +49094,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 9e-07, "supports_function_calling": true, @@ -50922,7 +49104,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50931,7 +49112,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.8e-07, "supports_function_calling": true, @@ -50943,7 +49123,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "supports_function_calling": true, @@ -50955,7 +49134,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true @@ -50965,7 +49143,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-08, "supports_function_calling": true, @@ -50976,7 +49153,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1e-07, "supports_vision": true, @@ -50988,7 +49164,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "chat", "output_cost_per_token": 0.0 }, @@ -50997,7 +49172,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -51008,7 +49182,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.9e-07 }, @@ -51017,7 +49190,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -51029,7 +49201,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 65536, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true @@ -51039,7 +49210,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_vision": true, @@ -51052,7 +49222,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 6e-06, "supports_vision": true, @@ -51065,7 +49234,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.2e-06, "supports_function_calling": true, @@ -51076,7 +49244,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06 }, @@ -51085,7 +49252,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.9e-06 }, @@ -51094,7 +49260,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 16385, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -51105,7 +49270,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 8192, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-06 }, @@ -51114,7 +49278,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 3e-05, "supports_vision": true, @@ -51128,7 +49291,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "supports_vision": true, @@ -51143,7 +49305,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.6e-06, "supports_vision": true, @@ -51158,7 +49319,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 1047576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4e-07, "supports_vision": true, @@ -51173,7 +49333,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_vision": true, @@ -51188,7 +49347,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, "supports_vision": true, @@ -51203,7 +49361,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_vision": true, @@ -51218,7 +49375,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_vision": true, @@ -51233,7 +49389,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_function_calling": true, @@ -51247,7 +49402,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, "supports_vision": true, @@ -51260,7 +49414,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -51269,7 +49422,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -51278,7 +49430,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 0, "max_output_tokens": 0, - "max_tokens": 0, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -51287,7 +49438,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-06 }, @@ -51296,7 +49446,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 200000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-05 }, @@ -51305,7 +49454,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 5e-06 }, @@ -51314,7 +49462,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 127000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-06 }, @@ -51323,7 +49470,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -51335,7 +49481,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_vision": true, @@ -51347,7 +49492,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 1e-05, "supports_function_calling": true, @@ -51358,7 +49502,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1e-05, "supports_vision": true, @@ -51370,7 +49513,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -51381,7 +49523,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.5e-05, "supports_function_calling": true @@ -51391,7 +49532,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "supports_function_calling": true, @@ -51402,7 +49542,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "supports_function_calling": true, @@ -51413,7 +49552,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_function_calling": true, @@ -51424,7 +49562,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-06, "supports_function_calling": true, @@ -51435,7 +49572,6 @@ "litellm_provider": "vercel_ai_gateway", "max_input_tokens": 128000, "max_output_tokens": 96000, - "max_tokens": 96000, "mode": "chat", "output_cost_per_token": 1.1e-06, "supports_function_calling": true, @@ -51447,7 +49583,6 @@ "input_cost_per_token": 4.5e-07, "max_input_tokens": 200000, "max_output_tokens": 200000, - "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 1.8e-06, "source": "https://vercel.com/ai-gateway/models/glm-4.6", @@ -51498,7 +49633,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -51512,7 +49646,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, @@ -51532,7 +49665,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -51561,7 +49693,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -51584,7 +49715,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51600,7 +49730,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51615,7 +49744,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, "supports_assistant_prefill": true, @@ -51629,7 +49757,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.25e-06, "supports_assistant_prefill": true, @@ -51643,7 +49770,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -51657,7 +49783,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 7.5e-05, "supports_assistant_prefill": true, @@ -51670,7 +49795,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51683,7 +49807,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.5e-05, "supports_assistant_prefill": true, @@ -51703,7 +49826,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51738,7 +49860,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51777,7 +49898,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51815,7 +49935,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51852,7 +49971,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51890,7 +50008,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -51928,7 +50045,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -51966,7 +50082,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -52010,7 +50125,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -52048,7 +50162,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "search_context_cost_per_query": { @@ -52093,7 +50206,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52132,7 +50244,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52169,7 +50280,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52210,7 +50320,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52253,7 +50362,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52292,7 +50400,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -52332,7 +50439,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -52362,7 +50468,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -52401,7 +50506,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -52440,7 +50544,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -52464,7 +50567,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52476,7 +50578,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52488,7 +50589,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52500,7 +50600,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -52512,7 +50611,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -52523,7 +50621,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -52534,7 +50631,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true, @@ -52548,7 +50644,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.7e-06, "output_cost_per_token_batches": 8.5e-07, @@ -52570,7 +50665,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.68e-06, "output_cost_per_token_batches": 8.4e-07, @@ -52591,7 +50685,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 65336, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "output_cost_per_token_batches": 2.7e-06, @@ -52614,7 +50707,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_image_token": 3e-05, @@ -52673,7 +50765,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -52708,7 +50799,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -52741,7 +50831,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -52769,7 +50858,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0672, "output_cost_per_image_token": 6e-05, @@ -52799,7 +50887,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -52839,7 +50926,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -52898,7 +50984,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 1.5e-06, "output_cost_per_token": 1.5e-06, @@ -52959,7 +51044,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -53016,7 +51100,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 65536, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_image": 0.134, "output_cost_per_image_token": 0.00012, @@ -53030,7 +51113,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -53041,7 +51123,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -53052,7 +51133,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "supports_tool_choice": true @@ -53063,7 +51143,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -53074,7 +51153,6 @@ "litellm_provider": "vertex_ai-ai21_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-07, "supports_tool_choice": true @@ -53104,7 +51182,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "audio_speech", "output_cost_per_image": 0.04, "output_cost_per_token": 0, @@ -53140,7 +51217,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "audio_speech", "output_cost_per_image": 0.08, "output_cost_per_token": 0, @@ -53177,7 +51253,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 1.6e-05, "source": "https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.2-90b-vision-instruct-maas", @@ -53190,7 +51265,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.2-90b-vision-instruct-maas", @@ -53203,7 +51277,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "metadata": { "notes": "VertexAI states that The Llama 3.1 API service for llama-3.1-70b-instruct-maas and llama-3.1-8b-instruct-maas are in public preview and at no cost." }, @@ -53219,7 +51292,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 2048, - "max_tokens": 2048, "metadata": { "notes": "VertexAI states that The Llama 3.2 API service is at no cost during public preview, and will be priced as per dollar-per-1M-tokens at GA." }, @@ -53236,7 +51308,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, "output_cost_per_token_batches": 5.75e-07, @@ -53258,7 +51329,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, "output_cost_per_token_batches": 5.75e-07, @@ -53280,7 +51350,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, - "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, "output_cost_per_token_batches": 3.5e-07, @@ -53302,7 +51371,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, - "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, "output_cost_per_token_batches": 3.5e-07, @@ -53323,7 +51391,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -53334,7 +51401,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -53345,7 +51411,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", @@ -53358,7 +51423,6 @@ "litellm_provider": "vertex_ai-minimax_models", "max_input_tokens": 196608, "max_output_tokens": 196608, - "max_tokens": 196608, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53372,7 +51436,6 @@ "litellm_provider": "vertex_ai-moonshot_models", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53387,7 +51450,6 @@ "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53405,7 +51467,6 @@ "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53422,7 +51483,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53434,7 +51494,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53446,7 +51505,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53458,7 +51516,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53470,7 +51527,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53481,7 +51537,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53492,7 +51547,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53503,7 +51557,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 6e-06, "supports_function_calling": true, @@ -53514,7 +51567,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -53525,7 +51577,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-07, "supports_function_calling": true, @@ -53536,7 +51587,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -53549,7 +51599,6 @@ "litellm_provider": "vertex_ai-mistral_models", "max_input_tokens": 32000, "max_output_tokens": 8191, - "max_tokens": 8191, "mode": "chat", "output_cost_per_token": 3e-07, "supports_function_calling": true, @@ -53583,7 +51632,6 @@ "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 262144, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/maas/google/gemma-4-26b-a4b-it", @@ -53600,7 +51648,6 @@ "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.6e-07, "output_cost_per_token_batches": 1.8e-07, @@ -53613,7 +51660,6 @@ "litellm_provider": "vertex_ai-openai_models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53628,7 +51674,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53645,7 +51690,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53665,7 +51709,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, - "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -53685,7 +51728,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 2000000, "max_output_tokens": 2000000, - "max_tokens": 2000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -53706,7 +51748,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 200000, "max_output_tokens": 200000, - "max_tokens": 200000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -53726,7 +51767,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -53745,7 +51785,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 8.8e-07, "output_cost_per_token_batches": 4.4e-07, @@ -53765,7 +51804,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_batches": 9e-07, @@ -53782,7 +51820,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53798,7 +51835,6 @@ "litellm_provider": "vertex_ai-qwen_models", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -53811,7 +51847,6 @@ "vertex_ai/veo-3.1-generate-preview": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -53826,7 +51861,6 @@ "vertex_ai/veo-3.1-fast-generate-preview": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -53843,7 +51877,6 @@ "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "output_cost_per_second_4k": 0.6, @@ -53859,7 +51892,6 @@ "deprecation_date": "2026-11-17", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -53875,7 +51907,6 @@ "vertex_ai/veo-3.1-lite-generate-001": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.05, "output_cost_per_second_1080p": 0.08, @@ -53893,7 +51924,6 @@ "litellm_provider": "voyage", "max_input_tokens": 16000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53902,7 +51932,6 @@ "litellm_provider": "voyage", "max_input_tokens": 8000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53911,7 +51940,6 @@ "litellm_provider": "voyage", "max_input_tokens": 8000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53921,7 +51949,6 @@ "litellm_provider": "voyage", "max_input_tokens": 4000, "max_output_tokens": 4000, - "max_tokens": 4000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53931,7 +51958,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53940,7 +51966,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0 }, @@ -53949,7 +51974,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53959,7 +51983,6 @@ "litellm_provider": "voyage", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://docs.voyageai.com/docs/pricing" @@ -53968,7 +51991,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "voyage", "max_input_tokens": 4000, - "max_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -53976,7 +51998,6 @@ "input_cost_per_token": 6e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -53984,7 +52005,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -53994,7 +52014,6 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54002,7 +52021,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54010,7 +52028,6 @@ "input_cost_per_token": 6e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54018,7 +52035,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54026,7 +52042,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54034,7 +52049,6 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54042,7 +52056,6 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "voyage", "max_input_tokens": 120000, - "max_tokens": 120000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54050,7 +52063,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54058,7 +52070,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54066,7 +52077,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -54076,7 +52086,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 16000, - "max_tokens": 16000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54084,7 +52093,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "voyage", "max_input_tokens": 4096, - "max_tokens": 4096, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54092,7 +52100,6 @@ "input_cost_per_token": 1e-07, "litellm_provider": "voyage", "max_input_tokens": 4000, - "max_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -54100,13 +52107,11 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0 }, "wandb/openai/gpt-oss-120b": { "supports_reasoning": true, - "max_tokens": 131072, "max_input_tokens": 131000, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -54117,7 +52122,6 @@ }, "wandb/openai/gpt-oss-20b": { "supports_reasoning": true, - "max_tokens": 131072, "max_input_tokens": 131000, "max_output_tokens": 131072, "input_cost_per_token": 3e-08, @@ -54127,7 +52131,6 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/moonshotai/Kimi-K2.5": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "cache_read_input_token_cost": 1e-07, @@ -54142,7 +52145,6 @@ "supports_vision": true }, "wandb/meta-llama/Llama-3.1-8B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 131000, "max_output_tokens": 128000, "input_cost_per_token": 2.2e-07, @@ -54153,7 +52155,6 @@ }, "wandb/deepseek-ai/DeepSeek-V3.1": { "supports_reasoning": true, - "max_tokens": 128000, "max_input_tokens": 161000, "max_output_tokens": 128000, "input_cost_per_token": 5.5e-07, @@ -54163,7 +52164,6 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "wandb/meta-llama/Llama-3.3-70B-Instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 7.1e-07, @@ -54177,7 +52177,6 @@ "litellm_provider": "watsonx", "max_input_tokens": 8192, "max_output_tokens": 1024, - "max_tokens": 1024, "mode": "chat", "output_cost_per_token": 2e-07, "supports_audio_input": false, @@ -54195,7 +52194,6 @@ "litellm_provider": "watsonx", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supports_audio_input": false, @@ -54209,7 +52207,6 @@ "supports_vision": false }, "watsonx/bigscience/mt0-xxl-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.908e-06, @@ -54222,7 +52219,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/bigscience/mt0-xxl": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.908e-06, @@ -54235,7 +52231,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/core42/jais-13b-chat": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 0.0005, @@ -54247,7 +52242,6 @@ "supports_vision": false }, "watsonx/google/flan-t5-xl-3b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 6e-07, @@ -54259,7 +52253,6 @@ "supports_vision": false }, "watsonx/ibm/granite-13b-chat-v2": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 6e-07, @@ -54271,7 +52264,6 @@ "supports_vision": false }, "watsonx/ibm/granite-13b-instruct-v2": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 6e-07, @@ -54283,7 +52275,6 @@ "supports_vision": false }, "watsonx/ibm/granite-3-3-8b-instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -54295,7 +52286,6 @@ "supports_vision": false }, "watsonx/ibm/granite-4-h-small": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 6.36e-08, @@ -54308,7 +52298,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/ibm/granite-guardian-3-2-2b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -54320,7 +52309,6 @@ "supports_vision": false }, "watsonx/ibm/granite-guardian-3-3-8b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -54332,7 +52320,6 @@ "supports_vision": false }, "watsonx/ibm/granite-ttm-1024-96-r2": { - "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, "input_cost_per_token": 3.8e-07, @@ -54344,7 +52331,6 @@ "supports_vision": false }, "watsonx/ibm/granite-ttm-1536-96-r2": { - "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, "input_cost_per_token": 3.8e-07, @@ -54356,7 +52342,6 @@ "supports_vision": false }, "watsonx/ibm/granite-ttm-512-96-r2": { - "max_tokens": 512, "max_input_tokens": 512, "max_output_tokens": 512, "input_cost_per_token": 3.8e-07, @@ -54368,7 +52353,6 @@ "supports_vision": false }, "watsonx/ibm/granite-vision-3-2-2b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -54380,7 +52364,6 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-2-11b-vision-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3.5e-07, @@ -54392,7 +52375,6 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-2-1b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1e-07, @@ -54404,7 +52386,6 @@ "supports_vision": false }, "watsonx/meta-llama/llama-3-2-3b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.5e-07, @@ -54416,7 +52397,6 @@ "supports_vision": false }, "watsonx/meta-llama/llama-3-2-90b-vision-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-06, @@ -54428,7 +52408,6 @@ "supports_vision": true }, "watsonx/meta-llama/llama-3-3-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 7.526e-07, @@ -54441,7 +52420,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 3.71e-07, @@ -54454,7 +52432,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 3.71e-07, @@ -54467,7 +52444,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/meta-llama/llama-guard-3-11b-vision": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3.5e-07, @@ -54479,7 +52455,6 @@ "supports_vision": true }, "watsonx/mistralai/mistral-medium-2505": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, @@ -54491,7 +52466,6 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-2503": { - "max_tokens": 32000, "max_input_tokens": 32000, "max_output_tokens": 32000, "input_cost_per_token": 1e-07, @@ -54503,7 +52477,6 @@ "supports_vision": false }, "watsonx/mistralai/mistral-small-3-1-24b-instruct-2503": { - "max_tokens": 16384, "max_input_tokens": 131072, "max_output_tokens": 16384, "input_cost_per_token": 1.06e-07, @@ -54516,7 +52489,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/mistralai/pixtral-12b-2409": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 3.5e-07, @@ -54528,7 +52500,6 @@ "supports_vision": true }, "watsonx/openai/gpt-oss-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.59e-07, @@ -54541,7 +52512,6 @@ "source": "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx" }, "watsonx/sdaia/allam-1-13b-instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1.8e-06, @@ -54578,7 +52548,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -54603,7 +52572,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -54627,7 +52595,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -54653,7 +52620,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -54681,7 +52647,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -54709,7 +52674,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, @@ -54733,7 +52697,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54755,7 +52718,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54777,7 +52739,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54799,7 +52760,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54821,7 +52781,6 @@ "litellm_provider": "xai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -54843,7 +52802,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-06, "output_cost_per_token_batches": 1.1e-06, @@ -54866,7 +52824,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_batches": 1.6e-06, "output_cost_per_token_priority": 5.6e-06, @@ -54889,7 +52846,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, @@ -55487,7 +53443,6 @@ } }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-480b-a35b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 4.5e-07, @@ -55497,7 +53452,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/flux-kontext-pro": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 4e-08, @@ -55506,7 +53460,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/SSD-1B": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -55515,7 +53468,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/chronos-hermes-13b-v2": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55524,7 +53476,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-13b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55533,7 +53484,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-13b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55542,7 +53492,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-13b-python": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55551,7 +53500,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-34b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55560,7 +53508,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-34b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55569,7 +53516,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-34b-python": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55578,7 +53524,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-70b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -55587,7 +53532,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-70b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -55596,7 +53540,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-70b-python": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -55605,7 +53548,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-7b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55614,7 +53556,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-7b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55623,7 +53564,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-llama-7b-python": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -55632,7 +53572,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/code-qwen-1p5-7b": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 2e-07, @@ -55641,7 +53580,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/codegemma-2b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -55650,7 +53588,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/codegemma-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -55659,7 +53596,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-671b-v2-p1": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1.2e-06, @@ -55668,7 +53604,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-llama-3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -55677,7 +53612,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-llama-70b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55686,7 +53620,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-llama-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55695,7 +53628,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-qwen-14b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55704,7 +53636,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/cogito-v1-preview-qwen-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55713,7 +53644,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-kontext-max": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 8e-08, @@ -55722,7 +53652,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/dbrx-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.2e-06, @@ -55731,7 +53660,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-1b-base": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 1e-07, @@ -55740,7 +53668,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-33b-instruct": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -55749,7 +53676,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-7b-base": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55758,7 +53684,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-7b-base-v1p5": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55767,7 +53692,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-7b-instruct-v1p5": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55776,7 +53700,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-v2-lite-base": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -55785,7 +53708,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-coder-v2-lite-instruct": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -55794,7 +53716,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-prover-v2": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1.2e-06, @@ -55803,7 +53724,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-0528-distill-qwen3-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55812,7 +53732,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-llama-70b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55821,7 +53740,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-llama-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55830,7 +53748,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-14b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55839,7 +53756,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-1p5b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -55848,7 +53764,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55857,7 +53772,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-r1-distill-qwen-7b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55866,7 +53780,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-v2-lite-chat": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 5e-07, @@ -55875,7 +53788,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/deepseek-v2p5": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.2e-06, @@ -55884,7 +53796,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/devstral-small-2505": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55893,7 +53804,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dobby-mini-unhinged-plus-llama-3-1-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -55902,7 +53812,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dobby-unhinged-llama-3-3-70b-new": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55911,7 +53820,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dolphin-2-9-2-qwen2-72b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55920,7 +53828,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/dolphin-2p6-mixtral-8x7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -55929,7 +53836,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ernie-4p5-21b-a3b-pt": { - "max_tokens": 4096, "max_input_tokens": 131072, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -55938,7 +53844,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ernie-4p5-300b-a47b-pt": { - "max_tokens": 4096, "max_input_tokens": 131072, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -55947,7 +53852,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/fare-20b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -55956,7 +53860,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/firefunction-v1": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -55965,7 +53868,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/firellava-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -55974,7 +53876,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/firesearch-ocr-v6": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -55983,7 +53884,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/fireworks-asr-large": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.0, @@ -55992,7 +53892,6 @@ "mode": "audio_transcription" }, "fireworks_ai/accounts/fireworks/models/fireworks-asr-v2": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 0.0, @@ -56001,7 +53900,6 @@ "mode": "audio_transcription" }, "fireworks_ai/accounts/fireworks/models/flux-1-dev": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56010,7 +53908,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-1-dev-controlnet-union": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-09, @@ -56019,7 +53916,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-1-dev-fp8": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 5e-10, @@ -56028,7 +53924,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/flux-1-schnell": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56037,7 +53932,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/flux-1-schnell-fp8": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 3.5e-10, @@ -56046,7 +53940,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/gemma-2b-it": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -56055,7 +53948,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma-3-27b-it": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56064,7 +53956,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56073,7 +53964,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma-7b-it": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56082,7 +53972,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gemma2-9b-it": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56091,7 +53980,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/glm-4p5v": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.2e-06, @@ -56101,7 +53989,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/gpt-oss-safeguard-120b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.2e-06, @@ -56110,7 +53997,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/gpt-oss-safeguard-20b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 5e-07, @@ -56119,7 +54005,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/hermes-2-pro-mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56128,7 +54013,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/internvl3-38b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56137,7 +54021,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/internvl3-78b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56146,7 +54029,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/internvl3-8b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -56155,7 +54037,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/japanese-stable-diffusion-xl": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -56164,7 +54045,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/kat-coder": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-07, @@ -56173,7 +54053,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/kat-dev-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56182,7 +54061,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/kat-dev-72b-exp": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56191,7 +54069,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-guard-2-8b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56200,7 +54077,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-guard-3-1b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56209,7 +54085,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-guard-3-8b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56218,7 +54093,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56227,7 +54101,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-13b-chat": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56236,7 +54109,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-70b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56245,7 +54117,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-70b-chat": { - "max_tokens": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048, "input_cost_per_token": 9e-07, @@ -56254,7 +54125,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-7b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56263,7 +54133,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v2-7b-chat": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56272,7 +54141,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 9e-07, @@ -56281,7 +54149,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct-hf": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 9e-07, @@ -56290,7 +54157,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-8b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56299,7 +54165,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3-8b-instruct-hf": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56308,7 +54173,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct-long": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56317,7 +54181,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56326,7 +54189,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct-1b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56335,7 +54197,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-nemotron-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56344,7 +54205,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p2-1b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56353,7 +54213,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p2-3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56362,7 +54221,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56371,7 +54229,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llamaguard-7b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56380,7 +54237,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/llava-yi-34b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -56389,7 +54245,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/minimax-m1-80k": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56398,7 +54253,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/minimax-m2": { - "max_tokens": 4096, "max_input_tokens": 196608, "max_output_tokens": 4096, "input_cost_per_token": 3e-07, @@ -56407,7 +54261,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ministral-3-14b-instruct-2512": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 2e-07, @@ -56416,7 +54269,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ministral-3-3b-instruct-2512": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 1e-07, @@ -56425,7 +54277,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/ministral-3-8b-instruct-2512": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 2e-07, @@ -56434,7 +54285,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56443,7 +54293,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-instruct-4k": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56452,7 +54301,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-instruct-v0p2": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56461,7 +54309,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-instruct-v3": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56470,7 +54317,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-7b-v0p2": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56479,7 +54325,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-large-3-fp8": { - "max_tokens": 256000, "max_input_tokens": 256000, "max_output_tokens": 256000, "input_cost_per_token": 1.2e-06, @@ -56488,7 +54333,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-nemo-base-2407": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -56497,7 +54341,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-nemo-instruct-2407": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -56506,7 +54349,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mistral-small-24b-instruct-2501": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56515,7 +54357,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 1.2e-06, @@ -56524,7 +54365,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 1.2e-06, @@ -56533,7 +54373,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56542,7 +54381,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56551,7 +54389,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mixtral-8x7b-instruct-hf": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56560,7 +54397,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/mythomax-l2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56569,7 +54405,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nemotron-nano-v2-12b-vl": { - "max_tokens": 4096, "max_input_tokens": 131072, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -56578,7 +54413,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-capybara-7b-v1p9": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56587,7 +54421,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-2-mixtral-8x7b-dpo": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 5e-07, @@ -56596,7 +54429,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-2-yi-34b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -56605,7 +54437,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-llama2-13b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56614,7 +54445,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-llama2-70b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -56623,7 +54453,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nous-hermes-llama2-7b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -56632,7 +54461,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nvidia-nemotron-nano-12b-v2": { - "max_tokens": 131072, "max_input_tokens": 128000, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56641,7 +54469,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/nvidia-nemotron-nano-9b-v2": { - "max_tokens": 131072, "max_input_tokens": 128000, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56650,7 +54477,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openchat-3p5-0106-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -56659,7 +54485,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openhermes-2-mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56668,7 +54493,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openhermes-2p5-mistral-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56677,7 +54501,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/openorca-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56686,7 +54509,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phi-2-3b": { - "max_tokens": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048, "input_cost_per_token": 1e-07, @@ -56695,7 +54517,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phi-3-mini-128k-instruct": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -56704,7 +54525,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phi-3-vision-128k-instruct": { - "max_tokens": 32064, "max_input_tokens": 32064, "max_output_tokens": 32064, "input_cost_per_token": 2e-07, @@ -56713,7 +54533,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phind-code-llama-34b-python-v1": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56722,7 +54541,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phind-code-llama-34b-v1": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56731,7 +54549,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/phind-code-llama-34b-v2": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 9e-07, @@ -56740,7 +54557,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/playground-v2-1024px-aesthetic": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -56749,7 +54565,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/playground-v2-5-1024px-aesthetic": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -56758,7 +54573,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/pythia-12b": { - "max_tokens": 2048, "max_input_tokens": 2048, "max_output_tokens": 2048, "input_cost_per_token": 2e-07, @@ -56767,7 +54581,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen-qwq-32b-preview": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56776,7 +54589,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen-v2p5-14b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56785,7 +54597,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen-v2p5-7b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56794,7 +54605,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen1p5-72b-chat": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56803,7 +54613,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56812,7 +54621,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-vl-2b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56821,7 +54629,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-vl-72b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56830,7 +54637,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2-vl-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56839,7 +54645,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-0p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56848,7 +54653,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-14b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2e-07, @@ -56857,7 +54661,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-1p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56866,7 +54669,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56875,7 +54677,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-32b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56884,7 +54685,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-72b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56893,7 +54693,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-72b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56902,7 +54701,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56911,7 +54709,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-0p5b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56920,7 +54717,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-0p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56929,7 +54725,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-14b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56938,7 +54733,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-14b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -56947,7 +54741,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-1p5b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56956,7 +54749,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-1p5b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -56965,7 +54757,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56974,7 +54765,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct-128k": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -56983,7 +54773,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct-32k-rope": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 9e-07, @@ -56992,7 +54781,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct-64k": { - "max_tokens": 65536, "max_input_tokens": 65536, "max_output_tokens": 65536, "input_cost_per_token": 9e-07, @@ -57001,7 +54789,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-3b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -57010,7 +54797,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-3b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1e-07, @@ -57019,7 +54805,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57028,7 +54813,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-coder-7b-instruct": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57037,7 +54821,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-math-72b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57046,7 +54829,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-32b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 9e-07, @@ -57055,7 +54837,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-3b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -57064,7 +54845,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-72b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 9e-07, @@ -57073,7 +54853,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen2p5-vl-7b-instruct": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -57082,7 +54861,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-0p6b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-07, @@ -57091,7 +54869,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-14b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57100,7 +54877,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -57109,7 +54885,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b-fp8-draft": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1e-07, @@ -57118,7 +54893,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b-fp8-draft-131072": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1e-07, @@ -57127,7 +54901,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-1p7b-fp8-draft-40960": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 1e-07, @@ -57136,7 +54909,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 2.2e-07, @@ -57145,7 +54917,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b-instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57154,7 +54925,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b-thinking-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57163,7 +54933,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-30b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.5e-07, @@ -57172,7 +54941,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-30b-a3b-instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 5e-07, @@ -57181,7 +54949,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-30b-a3b-thinking-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 9e-07, @@ -57190,7 +54957,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -57200,7 +54966,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/qwen3-4b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57209,7 +54974,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-4b-instruct-2507": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2e-07, @@ -57218,7 +54982,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-8b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57228,7 +54991,6 @@ "supports_reasoning": true }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-30b-a3b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -57237,7 +54999,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-coder-480b-instruct-bf16": { - "max_tokens": 4096, "max_input_tokens": 262144, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57246,7 +55007,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-0p6b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 0.0, @@ -57255,7 +55015,6 @@ "mode": "embedding" }, "fireworks_ai/accounts/fireworks/models/qwen3-embedding-4b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 0.0, @@ -57264,7 +55023,6 @@ "mode": "embedding" }, "fireworks_ai/accounts/fireworks/models/qwen3-next-80b-a3b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57273,7 +55031,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-next-80b-a3b-thinking": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57282,7 +55039,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-reranker-0p6b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 0.0, @@ -57291,7 +55047,6 @@ "mode": "rerank" }, "fireworks_ai/accounts/fireworks/models/qwen3-reranker-4b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 0.0, @@ -57300,7 +55055,6 @@ "mode": "rerank" }, "fireworks_ai/accounts/fireworks/models/qwen3-reranker-8b": { - "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, "input_cost_per_token": 2e-07, @@ -57310,7 +55064,6 @@ "source": "https://api.fireworks.ai/v1/serverless/models" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-235b-a22b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57319,7 +55072,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-235b-a22b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.2e-07, @@ -57328,7 +55080,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-30b-a3b-instruct": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -57337,7 +55088,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-30b-a3b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -57346,7 +55096,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-32b-instruct": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57355,7 +55104,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/qwen3-vl-8b-instruct": { - "max_tokens": 4096, "max_input_tokens": 262144, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -57369,7 +55117,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -57380,7 +55127,6 @@ "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/qwq-32b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-07, @@ -57389,7 +55135,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/rolm-ocr": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 2e-07, @@ -57398,7 +55143,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/snorkel-mistral-7b-pairrm-dpo": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57407,7 +55151,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/stable-diffusion-xl-1024-v1-0": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1.3e-10, @@ -57416,7 +55159,6 @@ "mode": "image_generation" }, "fireworks_ai/accounts/fireworks/models/stablecode-3b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 1e-07, @@ -57425,7 +55167,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder-16b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -57434,7 +55175,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder-7b": { - "max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192, "input_cost_per_token": 2e-07, @@ -57443,7 +55183,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder2-15b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -57452,7 +55191,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder2-3b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 1e-07, @@ -57461,7 +55199,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/starcoder2-7b": { - "max_tokens": 16384, "max_input_tokens": 16384, "max_output_tokens": 16384, "input_cost_per_token": 2e-07, @@ -57470,7 +55207,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/toppy-m-7b": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57479,7 +55215,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-34b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57488,7 +55223,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-34b-200k-capybara": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 9e-07, @@ -57497,7 +55231,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-34b-chat": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 9e-07, @@ -57506,7 +55239,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/yi-6b": { - "max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 4096, "input_cost_per_token": 2e-07, @@ -57515,7 +55247,6 @@ "mode": "chat" }, "fireworks_ai/accounts/fireworks/models/zephyr-7b-beta": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 2e-07, @@ -57529,7 +55260,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -57546,7 +55276,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -57563,7 +55292,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -57578,7 +55306,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -57590,7 +55317,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "supports_function_calling": true, @@ -57602,7 +55328,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 2.25e-06, "supports_function_calling": true @@ -57618,7 +55343,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, "supports_function_calling": true, @@ -57629,7 +55353,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "supports_function_calling": true @@ -57645,7 +55368,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 5e-07, "supports_function_calling": true, @@ -57657,7 +55379,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.5e-06, "supports_reasoning": true, @@ -57670,7 +55391,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.5e-07, "supports_function_calling": true, @@ -57681,7 +55401,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 2e-07, "supports_vision": true, @@ -57699,7 +55418,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 9e-07, "supports_function_calling": true @@ -57711,7 +55429,6 @@ "output_cost_per_token": 4e-07, "max_input_tokens": 163840, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57728,7 +55445,6 @@ "output_cost_per_token": 1.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57744,7 +55460,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57761,7 +55476,6 @@ "output_cost_per_token": 3.3e-07, "max_input_tokens": 262144, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57778,7 +55492,6 @@ "output_cost_per_token": 1.38e-07, "max_input_tokens": 65536, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_vision": true, "supports_system_messages": true }, @@ -57789,7 +55502,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, "max_output_tokens": 100352, - "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57806,7 +55518,6 @@ "output_cost_per_token": 1.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57823,7 +55534,6 @@ "output_cost_per_token": 2e-08, "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_vision": true, "supports_system_messages": true }, @@ -57834,7 +55544,6 @@ "output_cost_per_token": 4.1e-07, "max_input_tokens": 163840, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57849,7 +55558,6 @@ "output_cost_per_token": 3.95e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_vision": true, "supports_system_messages": true, "supports_reasoning": true, @@ -57863,7 +55571,6 @@ "output_cost_per_token": 9e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57881,7 +55588,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57898,7 +55604,6 @@ "output_cost_per_token": 1.2e-06, "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57914,7 +55619,6 @@ "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57929,7 +55633,6 @@ "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57944,7 +55647,6 @@ "output_cost_per_token": 3e-08, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_vision": true, "supports_system_messages": true, @@ -57957,7 +55659,6 @@ "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57974,7 +55675,6 @@ "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -57989,7 +55689,6 @@ "output_cost_per_token": 8.45e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58003,7 +55702,6 @@ "output_cost_per_token": 6e-07, "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_tool_choice": true, "supports_vision": true, "supports_system_messages": true, @@ -58016,7 +55714,6 @@ "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58033,7 +55730,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, "max_output_tokens": 100352, - "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58047,7 +55743,6 @@ "output_cost_per_token": 1.55e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58061,7 +55756,6 @@ "output_cost_per_token": 2.7e-07, "max_input_tokens": 160000, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58075,7 +55769,6 @@ "output_cost_per_token": 2.5e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58091,7 +55784,6 @@ "output_cost_per_token": 2.3e-06, "max_input_tokens": 131072, "max_output_tokens": 100352, - "max_tokens": 100352, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58105,7 +55797,6 @@ "output_cost_per_token": 1.12e-06, "max_input_tokens": 163840, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58121,7 +55812,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 131072, "max_output_tokens": 98304, - "max_tokens": 98304, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58137,7 +55827,6 @@ "output_cost_per_token": 3e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58151,7 +55840,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 16384, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_system_messages": true, "supports_response_schema": true }, @@ -58162,7 +55850,6 @@ "output_cost_per_token": 1e-07, "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_vision": true, "supports_system_messages": true, @@ -58175,7 +55862,6 @@ "output_cost_per_token": 1.8e-06, "max_input_tokens": 65536, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58193,7 +55879,6 @@ "output_cost_per_token": 1.5e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_tool_choice": true, "supports_vision": false, "supports_system_messages": true, @@ -58207,7 +55892,6 @@ "output_cost_per_token": 5.8e-07, "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58221,7 +55905,6 @@ "output_cost_per_token": 1.5e-07, "max_input_tokens": 32768, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, @@ -58234,7 +55917,6 @@ "output_cost_per_token": 4e-07, "max_input_tokens": 12288, "max_output_tokens": 12288, - "max_tokens": 12288, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58248,7 +55930,6 @@ "output_cost_per_token": 4e-07, "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58262,7 +55943,6 @@ "output_cost_per_token": 1.7e-07, "max_input_tokens": 60288, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58274,7 +55954,6 @@ "output_cost_per_token": 2.2e-06, "max_input_tokens": 1000000, "max_output_tokens": 40000, - "max_tokens": 40000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58289,7 +55968,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 163840, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58306,7 +55984,6 @@ "output_cost_per_token": 3e-07, "max_input_tokens": 64000, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, @@ -58319,7 +55996,6 @@ "output_cost_per_token": 4e-08, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_system_messages": true, "supports_response_schema": true }, @@ -58330,7 +56006,6 @@ "output_cost_per_token": 6.2e-07, "max_input_tokens": 65535, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_system_messages": true, "supports_response_schema": true }, @@ -58341,7 +56016,6 @@ "output_cost_per_token": 9e-08, "max_input_tokens": 128000, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_system_messages": true, "supports_reasoning": true }, @@ -58352,7 +56026,6 @@ "output_cost_per_token": 8e-07, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, @@ -58365,7 +56038,6 @@ "output_cost_per_token": 7.4e-07, "max_input_tokens": 8192, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58377,7 +56049,6 @@ "output_cost_per_token": 8e-07, "max_input_tokens": 40960, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true @@ -58389,7 +56060,6 @@ "output_cost_per_token": 8.5e-07, "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_vision": true, "supports_system_messages": true, "supports_response_schema": true @@ -58401,7 +56071,6 @@ "output_cost_per_token": 5.9e-07, "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_vision": true, "supports_system_messages": true }, @@ -58412,7 +56081,6 @@ "output_cost_per_token": 1.4e-07, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58424,7 +56092,6 @@ "output_cost_per_token": 8e-07, "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_vision": true, "supports_system_messages": true }, @@ -58435,7 +56102,6 @@ "output_cost_per_token": 1.48e-06, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58448,7 +56114,6 @@ "output_cost_per_token": 2.8e-07, "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_system_messages": true, "supports_reasoning": true }, @@ -58459,7 +56124,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58471,7 +56135,6 @@ "output_cost_per_token": 7e-08, "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-424b-a47b": { @@ -58481,7 +56144,6 @@ "output_cost_per_token": 1.25e-06, "max_input_tokens": 123000, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_vision": true, "supports_system_messages": true, "supports_reasoning": true @@ -58493,7 +56155,6 @@ "output_cost_per_token": 1.1e-06, "max_input_tokens": 123000, "max_output_tokens": 12000, - "max_tokens": 12000, "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true @@ -58505,7 +56166,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 160000, "max_output_tokens": 160000, - "max_tokens": 160000, "supports_system_messages": true }, "novita/qwen/qwen3-32b-fp8": { @@ -58515,7 +56175,6 @@ "output_cost_per_token": 4.5e-07, "max_input_tokens": 40960, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true }, @@ -58526,7 +56185,6 @@ "output_cost_per_token": 4.5e-07, "max_input_tokens": 40960, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true, "supports_function_calling": true, @@ -58539,7 +56197,6 @@ "output_cost_per_token": 2e-07, "max_input_tokens": 98304, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_vision": true, "supports_system_messages": true }, @@ -58550,7 +56207,6 @@ "output_cost_per_token": 1.3e-06, "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58563,7 +56219,6 @@ "output_cost_per_token": 2.5e-06, "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58578,7 +56233,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 8192, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58591,7 +56245,6 @@ "output_cost_per_token": 9e-08, "max_input_tokens": 4096, "max_output_tokens": 3200, - "max_tokens": 3200, "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-28b-a3b-thinking": { @@ -58601,7 +56254,6 @@ "output_cost_per_token": 3.9e-07, "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58617,7 +56269,6 @@ "output_cost_per_token": 5e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58632,7 +56283,6 @@ "output_cost_per_token": 8.5e-07, "max_input_tokens": 131072, "max_output_tokens": 98304, - "max_tokens": 98304, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58648,7 +56298,6 @@ "output_cost_per_token": 7e-07, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58663,7 +56312,6 @@ "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58678,7 +56326,6 @@ "output_cost_per_token": 9.7e-07, "max_input_tokens": 65536, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58695,7 +56342,6 @@ "output_cost_per_token": 9.7e-07, "max_input_tokens": 65536, "max_output_tokens": 16384, - "max_tokens": 16384, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58712,7 +56358,6 @@ "output_cost_per_token": 7.5e-07, "max_input_tokens": 16384, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-28b-a3b": { @@ -58722,7 +56367,6 @@ "output_cost_per_token": 5.6e-07, "max_input_tokens": 30000, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58737,7 +56381,6 @@ "output_cost_per_token": 2.8e-07, "max_input_tokens": 120000, "max_output_tokens": 8000, - "max_tokens": 8000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58750,7 +56393,6 @@ "output_cost_per_token": 1.38e-07, "max_input_tokens": 128000, "max_output_tokens": 20000, - "max_tokens": 20000, "supports_system_messages": true, "supports_reasoning": true }, @@ -58761,7 +56403,6 @@ "output_cost_per_token": 3e-08, "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_system_messages": true, "supports_reasoning": true, "supports_function_calling": true, @@ -58774,7 +56415,6 @@ "output_cost_per_token": 7e-08, "max_input_tokens": 32000, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58788,7 +56428,6 @@ "output_cost_per_token": 5e-08, "max_input_tokens": 32768, "max_output_tokens": 32000, - "max_tokens": 32000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58801,7 +56440,6 @@ "output_cost_per_token": 1.48e-06, "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_tool_choice": true, @@ -58813,8 +56451,7 @@ "input_cost_per_token": 7e-08, "output_cost_per_token": 0, "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "novita/qwen/qwen3-embedding-8b": { "litellm_provider": "novita", @@ -58822,8 +56459,7 @@ "input_cost_per_token": 7e-08, "output_cost_per_token": 0, "max_input_tokens": 32768, - "max_output_tokens": 4096, - "max_tokens": 4096 + "max_output_tokens": 4096 }, "novita/baai/bge-m3": { "litellm_provider": "novita", @@ -58831,8 +56467,7 @@ "input_cost_per_token": 1e-08, "output_cost_per_token": 1e-08, "max_input_tokens": 8192, - "max_output_tokens": 96000, - "max_tokens": 96000 + "max_output_tokens": 96000 }, "novita/qwen/qwen3-reranker-8b": { "litellm_provider": "novita", @@ -58840,8 +56475,7 @@ "input_cost_per_token": 5e-08, "output_cost_per_token": 5e-08, "max_input_tokens": 32768, - "max_output_tokens": 4096, - "max_tokens": 4096 + "max_output_tokens": 4096 }, "novita/baai/bge-reranker-v2-m3": { "litellm_provider": "novita", @@ -58849,11 +56483,9 @@ "input_cost_per_token": 1e-08, "output_cost_per_token": 1e-08, "max_input_tokens": 8000, - "max_output_tokens": 8000, - "max_tokens": 8000 + "max_output_tokens": 8000 }, "llamagate/llama-3.1-8b": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 3e-08, @@ -58864,7 +56496,6 @@ "supports_response_schema": true }, "llamagate/llama-3.2-3b": { - "max_tokens": 8192, "max_input_tokens": 131072, "max_output_tokens": 8192, "input_cost_per_token": 4e-08, @@ -58875,7 +56506,6 @@ "supports_response_schema": true }, "llamagate/mistral-7b-v0.3": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 1e-07, @@ -58886,7 +56516,6 @@ "supports_response_schema": true }, "llamagate/qwen3-8b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 4e-08, @@ -58897,7 +56526,6 @@ "supports_response_schema": true }, "llamagate/dolphin3-8b": { - "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "input_cost_per_token": 8e-08, @@ -58908,7 +56536,6 @@ "supports_response_schema": true }, "llamagate/deepseek-r1-8b": { - "max_tokens": 16384, "max_input_tokens": 65536, "max_output_tokens": 16384, "input_cost_per_token": 1e-07, @@ -58920,7 +56547,6 @@ "supports_reasoning": true }, "llamagate/deepseek-r1-7b-qwen": { - "max_tokens": 16384, "max_input_tokens": 131072, "max_output_tokens": 16384, "input_cost_per_token": 8e-08, @@ -58932,7 +56558,6 @@ "supports_reasoning": true }, "llamagate/openthinker-7b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 8e-08, @@ -58944,7 +56569,6 @@ "supports_reasoning": true }, "llamagate/qwen2.5-coder-7b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 6e-08, @@ -58955,7 +56579,6 @@ "supports_response_schema": true }, "llamagate/deepseek-coder-6.7b": { - "max_tokens": 4096, "max_input_tokens": 16384, "max_output_tokens": 4096, "input_cost_per_token": 6e-08, @@ -58966,7 +56589,6 @@ "supports_response_schema": true }, "llamagate/codellama-7b": { - "max_tokens": 4096, "max_input_tokens": 16384, "max_output_tokens": 4096, "input_cost_per_token": 6e-08, @@ -58977,7 +56599,6 @@ "supports_response_schema": true }, "llamagate/qwen3-vl-8b": { - "max_tokens": 8192, "max_input_tokens": 32768, "max_output_tokens": 8192, "input_cost_per_token": 1.5e-07, @@ -58989,7 +56610,6 @@ "supports_vision": true }, "llamagate/llava-7b": { - "max_tokens": 2048, "max_input_tokens": 4096, "max_output_tokens": 2048, "input_cost_per_token": 1e-07, @@ -59000,7 +56620,6 @@ "supports_vision": true }, "llamagate/gemma3-4b": { - "max_tokens": 8192, "max_input_tokens": 128000, "max_output_tokens": 8192, "input_cost_per_token": 3e-08, @@ -59012,7 +56631,6 @@ "supports_vision": true }, "llamagate/nomic-embed-text": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 2e-08, "output_cost_per_token": 0, @@ -59020,7 +56638,6 @@ "mode": "embedding" }, "llamagate/qwen3-embedding-8b": { - "max_tokens": 40960, "max_input_tokens": 40960, "input_cost_per_token": 2e-08, "output_cost_per_token": 0, @@ -59028,7 +56645,6 @@ "mode": "embedding" }, "libertai/hermes-3-8b-tee": { - "max_tokens": 16000, "max_input_tokens": 16000, "max_output_tokens": 16000, "input_cost_per_token": 1.5e-07, @@ -59042,7 +56658,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/gemma-4-31b-it": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59056,7 +56671,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/gemma-4-31b-it-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59071,7 +56685,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-27b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59085,7 +56698,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-27b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59100,7 +56712,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-35b-a3b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59114,7 +56725,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.6-35b-a3b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 1.5e-07, @@ -59129,7 +56739,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.5-122b-a10b": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -59143,7 +56752,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/qwen3.5-122b-a10b-thinking": { - "max_tokens": 262144, "max_input_tokens": 262144, "max_output_tokens": 262144, "input_cost_per_token": 2.5e-07, @@ -59158,7 +56766,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/deepseek-v4-flash": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 2.5e-07, @@ -59172,7 +56779,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/deepseek-v4-flash-thinking": { - "max_tokens": 200000, "max_input_tokens": 200000, "max_output_tokens": 200000, "input_cost_per_token": 2.5e-07, @@ -59187,7 +56793,6 @@ "source": "https://docs.libertai.io/apis/text/" }, "libertai/bge-m3": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 1e-08, "output_cost_per_token": 0.0, @@ -59203,7 +56808,6 @@ "litellm_provider": "sarvam", "max_input_tokens": 8192, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 0, "supports_reasoning": true @@ -59279,7 +56883,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -59307,7 +56910,6 @@ "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -59339,7 +56941,6 @@ "litellm_provider": "openai", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, @@ -59449,7 +57050,6 @@ "litellm_provider": "gemini", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_token": 0.0, @@ -59470,7 +57070,6 @@ "litellm_provider": "gemini", "max_input_tokens": 32768, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.039, "output_cost_per_token": 0.0, @@ -59493,7 +57092,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59519,7 +57117,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59546,7 +57143,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59581,7 +57177,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59616,7 +57211,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59650,7 +57244,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59683,7 +57276,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59711,7 +57303,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59740,7 +57331,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2e-06, @@ -59778,7 +57368,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -59815,7 +57404,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, @@ -59842,7 +57430,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 9e-06, "output_cost_per_token": 9e-06, @@ -59867,7 +57454,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 6e-06, "output_cost_per_token": 6e-06, @@ -59887,7 +57473,6 @@ "litellm_provider": "gemini", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, @@ -59907,7 +57492,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 3.75e-06, "output_cost_per_token": 3.75e-06, @@ -59966,7 +57550,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -60026,7 +57609,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -60088,7 +57670,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, @@ -60149,7 +57730,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65535, - "max_tokens": 65535, "mode": "chat", "output_cost_per_reasoning_token": 2.5e-06, "output_cost_per_token": 2.5e-06, @@ -60200,7 +57780,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -60239,7 +57818,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -60281,7 +57859,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -60307,7 +57884,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token_batches": 1.5e-07, "output_cost_per_token_priority": 5.25e-07, @@ -60333,7 +57909,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -60357,7 +57932,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 1e-07, "output_cost_per_token_priority": 3.5e-07, @@ -60387,7 +57961,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60431,7 +58004,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60466,7 +58038,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-cyber.html", "use_openai_responses_path": true, @@ -60502,7 +58073,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60542,7 +58112,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -60583,7 +58152,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html", "supported_modalities": [ @@ -60620,7 +58188,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-sol.html", "supported_modalities": [ @@ -60657,7 +58224,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", "supported_modalities": [ @@ -60694,7 +58260,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html", "supported_modalities": [ @@ -60731,7 +58296,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", "supported_modalities": [ @@ -60765,7 +58329,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", "supported_modalities": [ @@ -60797,7 +58360,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html", "supported_modalities": [ @@ -60829,7 +58391,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html", "supported_modalities": [ @@ -60861,7 +58422,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html", "supported_modalities": [ @@ -60896,7 +58456,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html", "supported_modalities": [ @@ -60937,7 +58496,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, "output_cost_per_token_ultrafast": 0.00033, @@ -60978,7 +58536,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61017,7 +58574,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61064,7 +58620,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, "output_cost_per_token_ultrafast": 0.00033, @@ -61105,7 +58660,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61144,7 +58698,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61189,7 +58742,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_above_272k_tokens_ultrafast": 0.00045, "output_cost_per_token_ultrafast": 0.0003, @@ -61229,7 +58781,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61264,7 +58815,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61302,7 +58852,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61337,7 +58886,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supported_modalities": [ "text", @@ -61377,7 +58925,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61417,7 +58964,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -61452,7 +58998,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 7e-07, @@ -61478,7 +59023,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token_batches": 2e-07, "output_cost_per_token_priority": 7e-07, @@ -61504,7 +59048,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token_batches": 4e-08, "output_cost_per_token_priority": 1.4e-07, @@ -61534,7 +59077,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1048576, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06, @@ -61562,7 +59104,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token_flex": 3.3e-06, "output_cost_per_token_priority": 1.155e-05, @@ -61587,7 +59128,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -61618,7 +59158,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.2e-05, "search_context_cost_per_query": { @@ -61656,7 +59195,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -61704,7 +59242,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token_flex": 3.3e-06, "output_cost_per_token_priority": 1.155e-05, @@ -61732,7 +59269,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token_flex": 3e-06, "output_cost_per_token_priority": 1.05e-05, @@ -61747,7 +59283,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -61787,7 +59322,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4.3125e-06, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61803,7 +59337,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 1024000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4.3125e-06, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61821,7 +59354,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 1024000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 3.88126e-07, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61838,7 +59370,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.15625e-06, "source": "https://www.volcengine.com/docs/82379/1544106", @@ -61852,7 +59383,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -61890,7 +59420,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -61928,7 +59457,6 @@ "litellm_provider": "volcengine", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "source": "https://docs.volcengine.com/docs/82379/1330310", "supports_function_calling": true, @@ -62023,7 +59551,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -62041,7 +59568,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, @@ -62062,7 +59588,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -62092,7 +59617,6 @@ "supports_tool_search": true, "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", @@ -62114,7 +59638,6 @@ } }, "snowflake/claude-sonnet-4-5": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62132,7 +59655,6 @@ "snowflake/claude-sonnet-4-6": { "supports_adaptive_thinking": true, "supports_legacy_thinking": true, - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62148,7 +59670,6 @@ "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-sonnet": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62164,7 +59685,6 @@ "prompt_cache_min_tokens": 1024 }, "snowflake/claude-4-opus": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 5e-06, @@ -62181,7 +59701,6 @@ "prompt_cache_min_tokens": 1024 }, "snowflake/claude-haiku-4-5": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 1e-06, @@ -62197,7 +59716,6 @@ "prompt_cache_min_tokens": 4096 }, "snowflake/claude-3-7-sonnet": { - "max_tokens": 16384, "max_input_tokens": 200000, "max_output_tokens": 16384, "input_cost_per_token": 3e-06, @@ -62213,7 +59731,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-4.1": { - "max_tokens": 16384, "max_input_tokens": 300000, "max_output_tokens": 16384, "input_cost_per_token": 2e-06, @@ -62228,7 +59745,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-5": { - "max_tokens": 16384, "max_input_tokens": 300000, "max_output_tokens": 16384, "input_cost_per_token": 1.25e-06, @@ -62244,7 +59760,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-5-mini": { - "max_tokens": 16384, "max_input_tokens": 1000000, "max_output_tokens": 16384, "input_cost_per_token": 3e-07, @@ -62256,7 +59771,6 @@ "supports_response_schema": true }, "snowflake/openai-gpt-5-nano": { - "max_tokens": 16384, "max_input_tokens": 5000000, "max_output_tokens": 16384, "input_cost_per_token": 1.5e-07, @@ -62268,7 +59782,6 @@ "supports_response_schema": true }, "snowflake/llama4-maverick": { - "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.4e-07, @@ -62279,7 +59792,6 @@ "supports_system_messages": true }, "snowflake/snowflake-arctic-embed-l-v2.0": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 7e-08, "output_cost_per_token": 0.0, @@ -62287,7 +59799,6 @@ "mode": "embedding" }, "snowflake/snowflake-arctic-embed-m-v2.0": { - "max_tokens": 8192, "max_input_tokens": 8192, "input_cost_per_token": 7e-08, "output_cost_per_token": 0.0, @@ -62297,7 +59808,7 @@ "soniox/stt-async-v4": { "litellm_provider": "soniox", "max_output_tokens": 8000, - "max_tokens": 8000, + "max_input_tokens": 8000, "input_cost_per_second": 0.0, "output_cost_per_second": 2.77778e-05, "mode": "audio_transcription", @@ -62310,7 +59821,7 @@ "soniox/stt-async-v5": { "litellm_provider": "soniox", "max_output_tokens": 8000, - "max_tokens": 8000, + "max_input_tokens": 8000, "input_cost_per_second": 0.0, "output_cost_per_second": 2.77778e-05, "mode": "audio_transcription", @@ -62487,7 +59998,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62541,7 +60051,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62595,7 +60104,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62649,7 +60157,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 2.2e-08, @@ -62703,7 +60210,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62757,7 +60263,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62811,7 +60316,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -62865,7 +60369,6 @@ "litellm_provider": "deepseek", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 2.2e-08, @@ -62919,7 +60422,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -62945,7 +60447,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -62971,7 +60472,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 2.2e-08, @@ -63026,7 +60526,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "off_peak_pricing": { "cache_read_input_token_cost": 3e-09, @@ -63081,7 +60580,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63106,7 +60604,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63131,7 +60628,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63155,7 +60651,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63179,7 +60674,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63203,7 +60697,6 @@ "litellm_provider": "tencent", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63227,7 +60720,6 @@ "litellm_provider": "tencent", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63251,7 +60743,6 @@ "litellm_provider": "tencent", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://www.tencentcloud.com/document/product/1300/78937", @@ -63318,7 +60809,6 @@ "source": "https://docs.devin.ai/desktop/models" }, "pinstripes/ps/glm-4.5-air": { - "max_tokens": 128000, "max_input_tokens": 128000, "max_output_tokens": 128000, "input_cost_per_token": 1.25e-07, @@ -63331,7 +60821,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3.6-35b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 1.4e-07, @@ -63344,7 +60833,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-30b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 9e-08, @@ -63357,7 +60845,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/qwen3-coder-30b-a3b": { - "max_tokens": 131072, "max_input_tokens": 131072, "max_output_tokens": 131072, "input_cost_per_token": 3e-07, @@ -63370,7 +60857,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/deepseek-v4-flash": { - "max_tokens": 163840, "max_input_tokens": 163840, "max_output_tokens": 163840, "input_cost_per_token": 1e-07, @@ -63383,7 +60869,6 @@ "source": "https://pinstripes.io/" }, "pinstripes/ps/minimax-m2.7": { - "max_tokens": 1000192, "max_input_tokens": 1000192, "max_output_tokens": 1000192, "input_cost_per_token": 2.55e-07, @@ -63400,7 +60885,6 @@ "litellm_provider": "darkbloom", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.65e-07, "source": "https://www.darkbloom.dev/", @@ -63417,7 +60901,6 @@ "litellm_provider": "darkbloom", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7e-08, "source": "https://www.darkbloom.dev/", @@ -63437,7 +60920,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -63464,7 +60946,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -63493,7 +60974,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.x.ai/v1/language-models", @@ -63566,7 +61046,6 @@ "litellm_provider": "openai", "max_input_tokens": 16000, "max_output_tokens": 2000, - "max_tokens": 2000, "mode": "realtime", "source": "https://developers.openai.com/api/docs/pricing", "supported_modalities": [ @@ -63591,7 +61070,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -63635,7 +61113,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -63678,7 +61155,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "prompt_cache_min_tokens": 512, @@ -63714,7 +61190,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 5e-06, "search_context_cost_per_query": { @@ -63749,7 +61224,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "reasoning_effort_levels": [ @@ -63770,7 +61244,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", @@ -63860,7 +61333,6 @@ "mode": "chat", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, @@ -63959,7 +61431,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 2.1e-05, "output_cost_per_token": 2.1e-05, @@ -63988,7 +61459,6 @@ "litellm_provider": "gemini", "max_input_tokens": 98304, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "audio_transcription", "output_cost_per_token": 1.2e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -64013,7 +61483,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "audio_transcription", "output_cost_per_token": 2.1e-05, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -64094,7 +61563,6 @@ "input_cost_per_token": 8e-09, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64104,7 +61572,6 @@ "input_cost_per_token": 5e-08, "litellm_provider": "perplexity", "max_input_tokens": 32768, - "max_tokens": 32768, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2560, @@ -64114,7 +61581,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64124,7 +61590,6 @@ "input_cost_per_token": 6e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64134,7 +61599,6 @@ "input_cost_per_token": 2e-08, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64144,7 +61608,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64154,7 +61617,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 120000, - "max_tokens": 120000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64164,7 +61626,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "voyage", "max_input_tokens": 32000, - "max_tokens": 32000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 1024, @@ -64180,7 +61641,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64199,7 +61659,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, @@ -64219,7 +61678,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64239,7 +61697,7 @@ "input_cost_per_token_priority": 2.75e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64257,7 +61715,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_priority": 1.875e-05, @@ -64282,7 +61739,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64301,7 +61757,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, @@ -64321,7 +61776,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64341,7 +61795,7 @@ "input_cost_per_token_priority": 2.75e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-07, "output_cost_per_token_priority": 8.25e-07, @@ -64358,7 +61812,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -64375,7 +61828,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -64393,7 +61845,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_priority": 1.875e-05, @@ -64415,7 +61866,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "reasoning_effort_levels": [ @@ -64438,7 +61888,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "output_cost_per_token_priority": 2.8125e-05, @@ -64480,7 +61929,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64500,7 +61948,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_priority": 2.5e-07, @@ -64520,7 +61967,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_priority": 3e-06, @@ -64540,7 +61986,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.5e-06, "output_cost_per_token_priority": 2.25e-06, @@ -64575,7 +62020,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "output_cost_per_token_priority": 2.5e-07, @@ -64595,7 +62039,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.4e-06, "output_cost_per_token_priority": 3e-06, @@ -64630,7 +62073,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -64647,7 +62089,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -64663,7 +62104,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "reasoning_effort_levels": [ @@ -64686,7 +62126,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.25e-05, "output_cost_per_token_priority": 2.8125e-05, @@ -64708,7 +62147,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -64725,7 +62163,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 3.96e-06, "source": "https://novita.ai/pricing", @@ -64742,7 +62179,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://novita.ai/pricing", @@ -64759,7 +62195,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 5.8e-07, "source": "https://novita.ai/pricing", @@ -64776,7 +62211,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://novita.ai/pricing", @@ -64793,7 +62227,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://novita.ai/pricing", @@ -64810,7 +62243,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -64827,7 +62259,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.32e-06, "source": "https://novita.ai/pricing", @@ -64844,7 +62275,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://novita.ai/pricing", @@ -64860,7 +62290,6 @@ "litellm_provider": "novita", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://novita.ai/pricing", @@ -64877,7 +62306,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://novita.ai/pricing", @@ -64894,7 +62322,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 3.2000000000000003e-06, "source": "https://novita.ai/pricing", @@ -64911,7 +62338,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://novita.ai/pricing", @@ -64927,7 +62353,6 @@ "litellm_provider": "novita", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://novita.ai/pricing", @@ -64944,7 +62369,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://novita.ai/pricing", @@ -64960,7 +62384,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.6e-06, "source": "https://novita.ai/pricing", @@ -64976,7 +62399,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 1.15e-06, "source": "https://novita.ai/pricing", @@ -64992,7 +62414,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 2.0000000000000002e-07, "source": "https://novita.ai/pricing", @@ -65008,7 +62429,6 @@ "litellm_provider": "novita", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.13e-06, "source": "https://novita.ai/pricing", @@ -65024,7 +62444,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.3600000000000004e-07, "source": "https://novita.ai/pricing", @@ -65041,7 +62460,6 @@ "litellm_provider": "novita", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://novita.ai/pricing", @@ -65058,7 +62476,6 @@ "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.044e-06, "source": "https://novita.ai/pricing", @@ -65074,7 +62491,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.6000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65090,7 +62506,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.4e-06, "source": "https://novita.ai/pricing", @@ -65107,7 +62522,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://novita.ai/pricing", @@ -65124,7 +62538,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65141,7 +62554,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65157,7 +62569,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.0000000000000003e-07, "source": "https://novita.ai/pricing", @@ -65172,7 +62583,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4.0000000000000003e-07, "source": "https://novita.ai/pricing", @@ -65188,7 +62598,6 @@ "litellm_provider": "novita", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65205,7 +62614,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://novita.ai/pricing", @@ -65222,7 +62630,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://novita.ai/pricing", @@ -65238,7 +62645,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://novita.ai/pricing", @@ -65253,7 +62659,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.2000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65268,7 +62673,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://novita.ai/pricing", @@ -65283,7 +62687,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.6000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65299,7 +62702,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://novita.ai/pricing", @@ -65316,7 +62718,6 @@ "litellm_provider": "novita", "max_input_tokens": 202800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.2000000000000003e-06, "source": "https://novita.ai/pricing", @@ -65332,7 +62733,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://novita.ai/pricing", @@ -65346,7 +62746,6 @@ "litellm_provider": "novita", "max_input_tokens": 8192, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-08, "source": "https://novita.ai/pricing", @@ -65358,7 +62757,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://novita.ai/pricing", @@ -65375,7 +62773,6 @@ "litellm_provider": "novita", "max_input_tokens": 204800, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65392,7 +62789,6 @@ "litellm_provider": "novita", "max_input_tokens": 200000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.0000000000000003e-07, "source": "https://novita.ai/pricing", @@ -65408,7 +62804,6 @@ "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.4850000000000002e-06, "source": "https://novita.ai/pricing", @@ -65423,7 +62818,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 8.900000000000001e-07, "source": "https://api.novita.ai/v3/openai/models", @@ -65436,7 +62830,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65450,7 +62843,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8.900000000000001e-07, "source": "https://api.novita.ai/v3/openai/models", @@ -65463,7 +62855,6 @@ "litellm_provider": "novita", "max_input_tokens": 64000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65477,7 +62868,6 @@ "litellm_provider": "novita", "max_input_tokens": 32000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.66e-06, "source": "https://api.novita.ai/v3/openai/models", @@ -65491,7 +62881,6 @@ "litellm_provider": "novita", "max_input_tokens": 131000, "max_output_tokens": 32000, - "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2e-08, "source": "https://api.novita.ai/v3/openai/models", @@ -65500,7 +62889,7 @@ }, "wandb/deepseek-ai/DeepSeek-V4-Flash": { "supports_reasoning": true, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1049000, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, @@ -65514,7 +62903,7 @@ }, "wandb/deepseek-ai/DeepSeek-V4-Flash-0731": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 1.3e-07, "output_cost_per_token": 2.8e-07, @@ -65527,7 +62916,7 @@ }, "wandb/deepseek-ai/DeepSeek-V4-Pro": { "supports_reasoning": true, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1049000, "input_cost_per_token": 1.15e-06, "output_cost_per_token": 2.55e-06, @@ -65541,7 +62930,7 @@ }, "wandb/google/gemma-4-31B-it": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3.4e-07, @@ -65552,7 +62941,7 @@ }, "wandb/ibm-granite/granite-4.1-8b": { "deprecation_date": "2026-10-05", - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, @@ -65563,7 +62952,7 @@ }, "wandb/JetBrains/Mellum2-12B-A2.5B-Instruct": { "deprecation_date": "2026-10-05", - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131000, "input_cost_per_token": 5e-08, "output_cost_per_token": 1e-07, @@ -65574,7 +62963,7 @@ }, "wandb/meta-llama/Llama-3.1-70B-Instruct": { "deprecation_date": "2026-10-05", - "max_tokens": 128000, + "max_output_tokens": 128000, "max_input_tokens": 131000, "input_cost_per_token": 8e-07, "output_cost_per_token": 8e-07, @@ -65585,7 +62974,7 @@ }, "wandb/MiniMaxAI/MiniMax-M3": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 2.3e-07, "output_cost_per_token": 9.6e-07, @@ -65598,7 +62987,7 @@ }, "wandb/moonshotai/Kimi-K2.7-Code": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 7.1e-07, "output_cost_per_token": 3.5e-06, @@ -65611,7 +63000,7 @@ }, "wandb/moonshotai/Kimi-K2.6": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 6.5e-07, "output_cost_per_token": 3.41e-06, @@ -65624,7 +63013,7 @@ }, "wandb/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 7e-08, "output_cost_per_token": 2e-07, @@ -65637,7 +63026,7 @@ }, "wandb/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.15e-06, @@ -65650,7 +63039,7 @@ }, "wandb/OpenPipe/Qwen3-14B-Instruct": { "deprecation_date": "2026-10-05", - "max_tokens": 32768, + "max_output_tokens": 32768, "max_input_tokens": 32800, "input_cost_per_token": 5e-08, "output_cost_per_token": 2.2e-07, @@ -65661,7 +63050,7 @@ }, "wandb/Qwen/Qwen3.8-27B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 4e-07, "output_cost_per_token": 3e-06, @@ -65674,7 +63063,7 @@ }, "wandb/Qwen/Qwen3.6-35B-A3B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.25e-06, @@ -65685,7 +63074,7 @@ }, "wandb/Qwen/Qwen3.6-27B": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 6e-07, "output_cost_per_token": 3.6e-06, @@ -65700,7 +63089,7 @@ "wandb/Qwen/Qwen3.5-35B-A3B": { "deprecation_date": "2026-10-05", "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.25e-06, @@ -65711,7 +63100,7 @@ }, "wandb/Qwen/Qwen3-30B-A3B-Instruct-2507": { "deprecation_date": "2026-10-05", - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262000, "input_cost_per_token": 1e-07, "output_cost_per_token": 3e-07, @@ -65744,7 +63133,7 @@ }, "wandb/zai-org/GLM-5.2": { "supports_reasoning": true, - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 1049000, "input_cost_per_token": 7.6e-07, "output_cost_per_token": 2.42e-06, @@ -65756,7 +63145,7 @@ "source": "https://wandb.ai/site/pricing/tokens/" }, "deepinfra/openai/gpt-oss-120b-Turbo": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, @@ -65770,7 +63159,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/MiniMaxAI/MiniMax-M2.7": { - "max_tokens": 196608, + "max_output_tokens": 196608, "max_input_tokens": 196608, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1e-06, @@ -65786,7 +63175,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.8-27B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4e-07, "output_cost_per_token": 3e-06, @@ -65802,7 +63191,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-31B-it-Ultra": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2.7e-07, "output_cost_per_token": 7.6e-07, @@ -65816,7 +63205,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K2.5": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4.5e-07, "output_cost_per_token": 2.25e-06, @@ -65832,7 +63221,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.7-Flash": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 6e-08, "output_cost_per_token": 4e-07, @@ -65848,7 +63237,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.6": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 5e-07, "output_cost_per_token": 2e-06, @@ -65867,7 +63256,7 @@ "input_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-05, "prompt_cache_min_tokens": 1024, @@ -65881,7 +63270,7 @@ "supports_vision": true }, "deepinfra/anthropic/claude-sonnet-4-6": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -65897,7 +63286,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemini-3.5-flash": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 1.5e-06, "output_cost_per_token": 9e-06, @@ -65912,7 +63301,7 @@ "supports_audio_input": true }, "deepinfra/XiaomiMiMo/MiMo-V2.5": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4e-07, "output_cost_per_token": 2e-06, @@ -65928,7 +63317,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Max": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1.2e-06, "output_cost_per_token": 6e-06, @@ -65943,7 +63332,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-31B-it-turbo": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 9e-08, "output_cost_per_token": 3.4e-07, @@ -65958,7 +63347,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/thinkingmachines/Inkling-Small": { - "max_tokens": 524288, + "max_output_tokens": 524288, "max_input_tokens": 524288, "input_cost_per_token": 4.5e-07, "output_cost_per_token": 1.2e-06, @@ -65974,7 +63363,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/meta-models/Muse-Glimmer-30B": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, @@ -65990,7 +63379,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-Max-Thinking": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1.2e-06, "output_cost_per_token": 6e-06, @@ -66005,7 +63394,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-VL-235B-A22B-Instruct": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2e-07, "output_cost_per_token": 8.8e-07, @@ -66020,7 +63409,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3-VL-30B-A3B-Instruct": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, @@ -66033,7 +63422,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-27B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.6e-07, "output_cost_per_token": 2.6e-06, @@ -66047,7 +63436,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.6-35B-A3B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, "output_cost_per_token": 9.5e-07, @@ -66061,7 +63450,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/nvidia/Nemotron-Content-Safety-3.5": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2e-07, "output_cost_per_token": 2e-07, @@ -66074,7 +63463,7 @@ "input_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-05, "prompt_cache_min_tokens": 512, @@ -66088,7 +63477,7 @@ "supports_vision": true }, "deepinfra/thinkingmachines/Inkling": { - "max_tokens": 524288, + "max_output_tokens": 524288, "max_input_tokens": 524288, "input_cost_per_token": 9.5e-07, "output_cost_per_token": 4.05e-06, @@ -66104,7 +63493,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K2.6": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 3.5e-06, @@ -66120,7 +63509,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Pro-0813": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.3e-06, "output_cost_per_token": 2.6e-06, @@ -66135,7 +63524,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.7-Max": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 2.5e-06, "output_cost_per_token": 7.5e-06, @@ -66150,7 +63539,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-2.0-mini": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1e-07, "output_cost_per_token": 4e-07, @@ -66166,7 +63555,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.8-2.4T-A95B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, @@ -66182,7 +63571,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/MiniMaxAI/MiniMax-M3": { - "max_tokens": 524288, + "max_output_tokens": 524288, "max_input_tokens": 524288, "input_cost_per_token": 2.8e-07, "output_cost_per_token": 1.1e-06, @@ -66198,7 +63587,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemini-3.1-flash-lite": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 1.5e-06, @@ -66213,7 +63602,7 @@ "supports_audio_input": true }, "deepinfra/google/gemini-3.7-flash": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 3.75e-06, @@ -66229,7 +63618,7 @@ "supports_audio_input": true }, "deepinfra/inclusionAI/Ling-3.0-flash": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 6e-08, "output_cost_per_token": 1.8e-07, @@ -66245,7 +63634,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/stepfun-ai/Step-3.7-Flash": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2e-07, "output_cost_per_token": 1.15e-06, @@ -66259,7 +63648,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-35B-A3B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 1e-06, @@ -66275,7 +63664,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-1.8": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06, @@ -66291,7 +63680,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/tencent/Hy3": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 5.8e-07, @@ -66307,7 +63696,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-2.0-code": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 5e-07, "output_cost_per_token": 3e-06, @@ -66323,7 +63712,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/ByteDance/Seed-2.0-pro": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 5e-07, "output_cost_per_token": 3e-06, @@ -66339,7 +63728,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-5": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 6e-07, "output_cost_per_token": 2.08e-06, @@ -66355,7 +63744,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/nvidia/Nemotron-3-Nano-30B-A3B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 5e-08, "output_cost_per_token": 2e-07, @@ -66370,7 +63759,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K2.7-Code": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 6.8e-07, "output_cost_per_token": 3.4e-06, @@ -66389,7 +63778,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1e-05, "prompt_cache_min_tokens": 1024, @@ -66403,7 +63792,7 @@ "supports_vision": true }, "deepinfra/Qwen/Qwen3.5-397B-A17B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 4.5e-07, "output_cost_per_token": 3e-06, @@ -66419,7 +63808,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 8e-08, "output_cost_per_token": 1.8e-07, @@ -66435,7 +63824,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-E4B-it": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2e-08, "output_cost_per_token": 1e-07, @@ -66448,7 +63837,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V3.2": { - "max_tokens": 163840, + "max_output_tokens": 163840, "max_input_tokens": 163840, "input_cost_per_token": 2.6e-07, "output_cost_per_token": 3.8e-07, @@ -66463,7 +63852,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.8-Max": { - "max_tokens": 256000, + "max_output_tokens": 256000, "max_input_tokens": 256000, "input_cost_per_token": 1.65e-06, "output_cost_per_token": 4.951e-06, @@ -66481,7 +63870,7 @@ "input_cost_per_token": 1e-05, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 5e-05, "prompt_cache_min_tokens": 512, @@ -66496,7 +63885,7 @@ "thinking_always_on": true }, "deepinfra/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.2e-06, @@ -66512,7 +63901,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-122B-A10B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 2.9e-07, "output_cost_per_token": 2.4e-06, @@ -66526,7 +63915,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-5.1": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 1.05e-06, "output_cost_per_token": 3.5e-06, @@ -66542,7 +63931,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Pro": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1.3e-06, "output_cost_per_token": 2.6e-06, @@ -66557,7 +63946,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 8.5e-08, "output_cost_per_token": 4e-07, @@ -66571,7 +63960,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-5.2": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 2.4e-06, @@ -66587,7 +63976,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/moonshotai/Kimi-K3": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 2.85e-06, "output_cost_per_token": 1.425e-05, @@ -66605,7 +63994,7 @@ "input_cost_per_token": 5e-06, "litellm_provider": "deepinfra", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-05, "prompt_cache_min_tokens": 2048, @@ -66619,7 +64008,7 @@ "supports_vision": true }, "deepinfra/Qwen/Qwen3.6-27B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 3.2e-07, "output_cost_per_token": 3.2e-06, @@ -66633,7 +64022,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-26B-A4B-it": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 7e-08, "output_cost_per_token": 3.4e-07, @@ -66647,7 +64036,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemini-3.1-pro": { - "max_tokens": 1000000, + "max_output_tokens": 1000000, "max_input_tokens": 1000000, "input_cost_per_token": 2e-06, "output_cost_per_token": 1.2e-05, @@ -66662,7 +64051,7 @@ "supports_audio_input": true }, "deepinfra/XiaomiMiMo/MiMo-V2.5-Pro": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 1e-06, "output_cost_per_token": 3e-06, @@ -66678,7 +64067,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/anthropic/claude-haiku-4-5": { - "max_tokens": 200000, + "max_output_tokens": 200000, "max_input_tokens": 200000, "input_cost_per_token": 1e-06, "output_cost_per_token": 5e-06, @@ -66693,7 +64082,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4-Flash": { - "max_tokens": 1048576, + "max_output_tokens": 1048576, "max_input_tokens": 1048576, "input_cost_per_token": 9e-08, "output_cost_per_token": 1.8e-07, @@ -66709,7 +64098,6 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/deepseek-ai/DeepSeek-V4.1-Flash": { - "max_tokens": 131072, "max_input_tokens": 1048576, "max_output_tokens": 131072, "input_cost_per_token": 1.4e-07, @@ -66726,7 +64114,6 @@ "source": "https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash" }, "deepinfra/deepseek-ai/DeepSeek-V4.1-Flash-1017": { - "max_tokens": 131072, "max_input_tokens": 1048576, "max_output_tokens": 131072, "input_cost_per_token": 1.4e-07, @@ -66743,7 +64130,7 @@ "source": "https://deepinfra.com/deepseek-ai/DeepSeek-V4.1-Flash-1017" }, "deepinfra/openai/gpt-oss-120b-Ultra": { - "max_tokens": 131072, + "max_output_tokens": 131072, "max_input_tokens": 131072, "input_cost_per_token": 2e-07, "output_cost_per_token": 9.5e-07, @@ -66757,7 +64144,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/Qwen/Qwen3.5-9B": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1e-07, "output_cost_per_token": 1.5e-07, @@ -66770,7 +64157,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/MiniMaxAI/MiniMax-M2.7-Turbo": { - "max_tokens": 196608, + "max_output_tokens": 196608, "max_input_tokens": 196608, "input_cost_per_token": 3.8e-07, "output_cost_per_token": 1.7e-06, @@ -66786,7 +64173,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/zai-org/GLM-4.7": { - "max_tokens": 202752, + "max_output_tokens": 202752, "max_input_tokens": 202752, "input_cost_per_token": 4e-07, "output_cost_per_token": 1.75e-06, @@ -66802,7 +64189,7 @@ "source": "https://deepinfra.com/pricing" }, "deepinfra/google/gemma-4-31B-it": { - "max_tokens": 262144, + "max_output_tokens": 262144, "max_input_tokens": 262144, "input_cost_per_token": 1.3e-07, "output_cost_per_token": 3.8e-07, @@ -66821,7 +64208,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -66856,7 +64242,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -66884,7 +64269,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -66912,7 +64296,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67188,7 +64571,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67215,7 +64597,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67242,7 +64623,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67273,7 +64653,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -67301,7 +64680,6 @@ "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://console.groq.com/docs/model/qwen/qwen3.8-27b", @@ -67317,7 +64695,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -67338,7 +64715,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -67359,7 +64735,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 7.5e-06, "reasoning_effort_levels": [ @@ -67380,7 +64755,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6e-07, "reasoning_effort_levels": [ @@ -67401,7 +64775,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -67416,7 +64789,6 @@ "litellm_provider": "mistral", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://docs.mistral.ai/models/model-cards/codestral-25-08", @@ -67431,7 +64803,6 @@ "litellm_provider": "mistral", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://mistral.ai/news/devstral-2-vibe-cli", @@ -67532,7 +64903,6 @@ "litellm_provider": "mistral", "max_input_tokens": 262144, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://docs.mistral.ai/models/model-cards/leanstral-1-5", @@ -67548,7 +64918,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -67567,7 +64936,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.6e-06, "output_cost_per_token_priority": 8.25e-06, @@ -67586,7 +64954,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.4e-06, "output_cost_per_token_priority": 5.5e-06, @@ -67605,7 +64972,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.6e-06, "output_cost_per_token_priority": 8.25e-06, @@ -67621,7 +64987,7 @@ "input_cost_per_token": 2.1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -67663,7 +65029,7 @@ "input_cost_per_token": 2.1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://docs.fireworks.ai/serverless/pricing", @@ -67680,7 +65046,7 @@ "input_cost_per_token_priority": 1.875e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_priority": 6.25e-07, @@ -67698,7 +65064,7 @@ "input_cost_per_token_priority": 2.8125e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 7.5e-07, "output_cost_per_token_priority": 9.375e-07, @@ -67715,7 +65081,7 @@ "input_cost_per_token_priority": 1.875e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_priority": 6.25e-07, @@ -67733,7 +65099,7 @@ "input_cost_per_token_priority": 2.8125e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 7.5e-07, "output_cost_per_token_priority": 9.375e-07, @@ -67751,7 +65117,7 @@ "input_cost_per_token_priority": 1e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.05e-06, "output_cost_per_token_priority": 4.05e-06, @@ -67767,7 +65133,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 40960, "max_output_tokens": 40960, - "max_tokens": 40960, "mode": "embedding", "source": "https://api.fireworks.ai/v1/serverless/models" }, @@ -67790,7 +65155,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 1000000, - "max_tokens": 1000000, + "max_output_tokens": 1000000, "mode": "chat", "output_cost_per_token": 4.7e-07, "source": "https://api.together.ai/v1/models" @@ -67850,7 +65215,6 @@ "litellm_provider": "cerebras", "max_input_tokens": 131072, "max_output_tokens": 40960, - "max_tokens": 40960, "mode": "chat", "output_cost_per_token": 1.49e-06, "source": "https://api.cerebras.ai/public/v1/models/gemma-4-31b", @@ -67876,7 +65240,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", @@ -67890,7 +65253,6 @@ "litellm_provider": "scaleway", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://www.scaleway.com/en/pricing/model-as-a-service/", @@ -67906,7 +65268,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -67929,7 +65290,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, "supports_function_calling": true, @@ -67971,7 +65331,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -67986,7 +65345,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.1e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68001,7 +65359,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68016,7 +65373,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.25e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68032,7 +65388,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 131072, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6.6e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68049,7 +65404,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68062,7 +65416,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -68137,7 +65490,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68155,7 +65507,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", @@ -68171,7 +65522,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", @@ -68186,7 +65536,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68204,7 +65553,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", @@ -68224,7 +65572,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", @@ -68249,7 +65596,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, @@ -68288,7 +65634,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, @@ -68326,7 +65671,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, @@ -68363,7 +65707,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -68401,7 +65744,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_adaptive_thinking": true, @@ -68436,7 +65778,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.88e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68454,7 +65795,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", @@ -68470,7 +65810,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", @@ -68485,7 +65824,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.8e-07, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -68503,7 +65841,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", @@ -68523,7 +65860,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", @@ -68548,7 +65884,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "prompt_cache_min_tokens": 1024, @@ -68582,7 +65917,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 1024, @@ -68615,7 +65949,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, @@ -68647,7 +65980,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -68680,7 +66012,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "supports_adaptive_thinking": true, @@ -68709,7 +66040,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68745,7 +66075,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68781,7 +66110,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68812,7 +66140,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1048576, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -68836,7 +66163,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -68855,7 +66181,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68874,7 +66199,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68893,7 +66217,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "use_openai_responses_path": true, "supported_endpoints": [ @@ -68913,7 +66236,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ @@ -68933,7 +66255,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ @@ -68956,7 +66277,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.4e-05, "prompt_cache_min_tokens": 512, @@ -68992,7 +66312,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -69021,7 +66340,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "use_openai_responses_path": true, "supported_endpoints": [ @@ -69054,7 +66372,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions", @@ -69074,7 +66391,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ @@ -69094,7 +66410,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ @@ -69118,7 +66433,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 8.4e-07, "output_cost_per_token_priority": 2.94e-06, @@ -69140,7 +66454,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_priority": 4.38e-06, @@ -69162,7 +66475,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 4.4e-07, "output_cost_per_token_priority": 1.54e-06, @@ -69184,7 +66496,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 32000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -69207,7 +66518,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 3e-07, "output_cost_per_token_priority": 1.05e-06, @@ -69229,7 +66539,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 262144, "max_output_tokens": 16000, - "max_tokens": 16000, "mode": "chat", "output_cost_per_token_batches": 9e-07, "output_cost_per_token_priority": 3.15e-06, @@ -69250,7 +66559,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 6e-07, "output_cost_per_token_priority": 2.1e-06, @@ -69272,7 +66580,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, - "max_tokens": 8000, "mode": "chat", "output_cost_per_token_batches": 1.33e-06, "output_cost_per_token_priority": 4.66e-06, @@ -69291,7 +66598,6 @@ "litellm_provider": "azure", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.375e-05, "supported_endpoints": [ @@ -69325,7 +66631,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 6.05e-06, "supports_prompt_caching": true, @@ -69338,7 +66643,6 @@ "input_cost_per_token": 1.63e-07, "litellm_provider": "azure", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -69346,7 +66650,6 @@ "input_cost_per_token": 2.5e-08, "litellm_provider": "azure", "max_input_tokens": 8191, - "max_tokens": 8191, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -69377,7 +66680,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_image": 0.04, "output_cost_per_token": 0, @@ -69402,7 +66704,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -69427,7 +66728,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/pricing", @@ -69452,7 +66752,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 0, "source": "https://ai.google.dev/gemini-api/docs/models/lyria-realtime-exp", @@ -69825,7 +67124,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -69849,7 +67147,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://inference.baseten.co/v1/models", @@ -69874,7 +67171,7 @@ "output_cost_per_token": 1.5e-07, "source": "https://api.together.ai/v1/models", "max_input_tokens": 128000, - "max_tokens": 128000 + "max_output_tokens": 128000 }, "vertex_ai/gemini-2.5-flash-native-audio": { "deprecation_date": "2026-12-13", @@ -69935,7 +67232,7 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai", "max_output_tokens": 57920, - "max_tokens": 57920, + "max_input_tokens": 57920, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, @@ -69998,7 +67295,7 @@ "output_cost_per_token": 3.5e-06, "source": "https://api.together.ai/v1/models", "max_input_tokens": 4096, - "max_tokens": 4096 + "max_output_tokens": 4096 }, "together_ai/meta-llama/Llama-3.2-1B-Instruct": { "input_cost_per_token": 6e-08, @@ -70007,7 +67304,7 @@ "output_cost_per_token": 6e-08, "source": "https://api.together.ai/v1/models", "max_input_tokens": 131072, - "max_tokens": 131072 + "max_output_tokens": 131072 }, "together_ai/meta-llama/Llama-3.2-3B-Instruct": { "input_cost_per_token": 6e-08, @@ -70016,7 +67313,7 @@ "output_cost_per_token": 6e-08, "source": "https://api.together.ai/v1/models", "max_input_tokens": 131072, - "max_tokens": 131072 + "max_output_tokens": 131072 }, "together_ai/Qwen/Qwen2-1.5B-Instruct": { "input_cost_per_token": 2e-08, @@ -70025,7 +67322,7 @@ "output_cost_per_token": 2e-08, "source": "https://api.together.ai/v1/models", "max_input_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "together_ai/Qwen/Qwen2.5-14B-Instruct": { "input_cost_per_token": 8e-07, @@ -70034,7 +67331,7 @@ "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models", "max_input_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "together_ai/Qwen/Qwen2.5-72B-Instruct": { "input_cost_per_token": 1.2e-06, @@ -70043,14 +67340,13 @@ "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", "max_input_tokens": 32768, - "max_tokens": 32768 + "max_output_tokens": 32768 }, "together_ai/Salesforce/Llama-Rank-V1": { "deprecation_date": "2026-02-06", "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, "mode": "rerank", "output_cost_per_token": 0.0, "source": "https://api.together.ai/v1/models" @@ -70059,7 +67355,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 16384, "mode": "completion", "output_cost_per_token": 2e-07, "source": "https://api.together.xyz/v1/models" @@ -70077,7 +67373,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://api.together.ai/v1/models" @@ -70087,7 +67383,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70097,7 +67393,7 @@ "input_cost_per_token": 9e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://api.together.ai/v1/models" @@ -70107,7 +67403,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70117,7 +67413,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70127,7 +67423,7 @@ "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "max_input_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models" @@ -70137,7 +67433,7 @@ "input_cost_per_token": 1.95e-06, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://api.together.ai/v1/models" @@ -70147,7 +67443,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.together.ai/v1/models" @@ -70157,7 +67453,7 @@ "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" @@ -70167,7 +67463,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models" @@ -70177,7 +67473,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models" @@ -70187,7 +67483,7 @@ "input_cost_per_token": 5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-06, "source": "https://api.together.ai/v1/models" @@ -70197,7 +67493,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 6.8e-07, "source": "https://api.together.ai/v1/models" @@ -70208,7 +67504,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.6e-06, "source": "https://api.together.ai/v1/models" @@ -70218,7 +67514,7 @@ "input_cost_per_token": 2e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.together.ai/v1/models" @@ -70228,7 +67524,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://api.together.ai/v1/models" @@ -70238,7 +67534,7 @@ "input_cost_per_token": 1.6e-06, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.6e-06, "source": "https://api.together.ai/v1/models" @@ -70248,7 +67544,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.7e-06, "source": "https://api.together.ai/v1/models" @@ -70258,7 +67554,7 @@ "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "max_input_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models" @@ -70268,7 +67564,7 @@ "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 8e-07, "source": "https://api.together.ai/v1/models" @@ -70278,7 +67574,7 @@ "input_cost_per_token": 3.9e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 9.7e-07, "source": "https://api.together.ai/v1/models" @@ -70288,7 +67584,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70298,7 +67594,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5.9e-07, "source": "https://api.together.ai/v1/models" @@ -70308,7 +67604,7 @@ "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://api.together.ai/v1/models" @@ -70318,7 +67614,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 8192, - "max_tokens": 8192, + "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70328,7 +67624,7 @@ "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://api.together.ai/v1/models" @@ -70338,7 +67634,7 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.8e-07, "source": "https://api.together.ai/v1/models" @@ -70348,7 +67644,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70358,7 +67654,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://api.together.ai/v1/models" @@ -70368,7 +67664,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 6e-07, "source": "https://api.together.ai/v1/models" @@ -70379,7 +67675,7 @@ "input_cost_per_token": 1.2e-06, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://api.together.ai/v1/models" @@ -70390,7 +67686,7 @@ "input_cost_per_token": 9.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://api.together.ai/v1/models" @@ -70400,7 +67696,7 @@ "input_cost_per_token": 8.8e-07, "litellm_provider": "together_ai", "max_input_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 32768, "mode": "chat", "output_cost_per_token": 8.8e-07, "source": "https://api.together.ai/v1/models" @@ -70411,7 +67707,7 @@ "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", "max_input_tokens": 512288, - "max_tokens": 512288, + "max_output_tokens": 512288, "mode": "chat", "output_cost_per_token": 3.6e-06, "source": "https://api.together.ai/v1/models" @@ -70421,7 +67717,7 @@ "input_cost_per_token": 5e-08, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 2e-07, "source": "https://api.together.ai/v1/models" @@ -70431,7 +67727,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "together_ai", "max_input_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.1e-06, "source": "https://api.together.ai/v1/models" @@ -70441,7 +67737,7 @@ "input_cost_per_token": 4.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://api.together.ai/v1/models" @@ -70451,7 +67747,7 @@ "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://api.together.ai/v1/models" @@ -70462,7 +67758,7 @@ "input_cost_per_token": 1.4e-06, "litellm_provider": "together_ai", "max_input_tokens": 202752, - "max_tokens": 202752, + "max_output_tokens": 202752, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://api.together.ai/v1/models" @@ -70529,7 +67825,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_batches": 8.25e-06, @@ -70702,7 +67997,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-05, "output_cost_per_token_above_272k_tokens": 9e-05, @@ -70746,7 +68040,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_above_272k_tokens": 9e-07, @@ -70795,7 +68088,6 @@ "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_272k_tokens": 1.8e-05, @@ -70851,7 +68143,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -70865,7 +68156,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 8.8e-06, "output_cost_per_token_batches": 4.4e-06, @@ -70888,7 +68178,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -70902,7 +68191,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, - "max_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.84e-06, "output_cost_per_token_batches": 2.42e-06, @@ -70937,7 +68225,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -70959,7 +68246,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 4.5e-06, @@ -71035,7 +68321,6 @@ "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_batches": 8.25e-06, @@ -71248,7 +68533,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, @@ -71296,7 +68580,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "audio_transcription", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ @@ -71337,7 +68620,6 @@ "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, - "max_tokens": 4096, "mode": "realtime", "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ @@ -71355,7 +68637,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 512000, "max_output_tokens": 65500, - "max_tokens": 65500, "mode": "chat", "output_cost_per_token": 1.5e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71368,7 +68649,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71381,7 +68661,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 200000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71396,7 +68675,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71417,7 +68695,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5.5e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71435,7 +68712,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71455,7 +68731,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71476,7 +68751,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71495,7 +68769,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.2e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71510,7 +68783,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.61333e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71539,7 +68811,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.6e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71555,7 +68826,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 4e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71569,7 +68839,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 128000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71584,7 +68853,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.84e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71599,7 +68867,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.38e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71614,7 +68881,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.411e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71630,7 +68896,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.4246e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71647,7 +68912,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3.0136e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71664,7 +68928,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.411e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71681,7 +68944,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2.32375e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71698,7 +68960,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 119000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.5353e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71711,7 +68972,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71728,7 +68988,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71745,7 +69004,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71762,7 +69020,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71779,7 +69036,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.499999e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71797,7 +69053,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.75e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71814,7 +69069,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 3.9998e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71826,7 +69080,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 131100, - "max_tokens": 131100, "mode": "chat", "output_cost_per_token": 3.9998e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71839,7 +69092,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 7.889e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71854,7 +69106,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.9438e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71869,7 +69120,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.9438e-07, "source": "https://aihubmix.com/api/v1/models", @@ -71885,7 +69135,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 200000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.09848e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71899,7 +69148,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.4e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71915,7 +69163,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71932,7 +69179,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71949,7 +69195,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4.5e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71966,7 +69211,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.25e-06, "source": "https://aihubmix.com/api/v1/models", @@ -71983,7 +69227,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "source": "https://aihubmix.com/api/v1/models", @@ -71999,7 +69242,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 0.00018, "source": "https://aihubmix.com/api/v1/models", @@ -72016,7 +69258,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72033,7 +69274,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72050,7 +69290,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1050000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72066,7 +69305,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 400000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 3e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72111,7 +69349,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72127,7 +69364,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72143,7 +69379,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.248e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72159,7 +69394,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.535e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72175,7 +69409,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 3.9995e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72191,7 +69424,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 7.999e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72207,7 +69439,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://aihubmix.com/api/v1/models", @@ -72223,7 +69454,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.0984e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72236,7 +69466,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 8e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72272,7 +69501,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 204800, "max_output_tokens": 204800, - "max_tokens": 204800, "mode": "chat", "output_cost_per_token": 1.1832e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72286,7 +69514,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 1.152e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72311,7 +69538,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 5.48e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72323,7 +69549,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9.008e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72338,7 +69563,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 9.864e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72353,7 +69577,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.532e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72368,7 +69591,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.524e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72385,7 +69607,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 262144, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 7.608e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72402,7 +69623,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.128e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72419,7 +69639,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72436,7 +69655,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 3.80025e-07, "source": "https://aihubmix.com/api/v1/models", @@ -72454,7 +69672,6 @@ "litellm_provider": "aihubmix", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5.07e-06, "source": "https://aihubmix.com/api/v1/models", @@ -72639,7 +69856,7 @@ "input_cost_per_token": 2e-07, "litellm_provider": "wandb", "max_input_tokens": 1049000, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6.5e-07, "source": "https://wandb.ai/site/pricing/tokens/", @@ -72652,7 +69869,7 @@ "input_cost_per_token": 1e-07, "litellm_provider": "wandb", "max_input_tokens": 262000, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-07, "source": "https://wandb.ai/site/pricing/tokens/", @@ -72693,7 +69910,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, @@ -72725,7 +69941,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_flex": 7.5e-06, @@ -72757,7 +69972,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, @@ -72779,7 +69993,6 @@ "litellm_provider": "xiaomi_mimo", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8.7e-07, "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md", @@ -72802,7 +70015,6 @@ "litellm_provider": "xiaomi_mimo", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.8e-07, "source": "https://platform.xiaomimimo.com/static/docs/price/pay-as-you-go.md", @@ -72827,7 +70039,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, @@ -72853,7 +70064,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72875,7 +70085,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72897,7 +70106,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72919,7 +70127,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72940,7 +70147,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72962,7 +70168,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -72983,7 +70188,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73005,7 +70209,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73027,7 +70230,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73048,7 +70250,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73070,7 +70271,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73092,7 +70292,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73113,7 +70312,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73135,7 +70333,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73160,7 +70357,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73185,7 +70381,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73210,7 +70405,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73231,7 +70425,6 @@ "litellm_provider": "xai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, - "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://api.x.ai/v1/language-models", @@ -73253,7 +70446,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -73271,7 +70463,6 @@ "litellm_provider": "baseten", "max_input_tokens": 262000, "max_output_tokens": 262000, - "max_tokens": 262000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73289,7 +70480,6 @@ "litellm_provider": "baseten", "max_input_tokens": 262000, "max_output_tokens": 262000, - "max_tokens": 262000, "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73306,7 +70496,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://inference.baseten.co/v1/models", @@ -73323,7 +70512,6 @@ "litellm_provider": "baseten", "max_input_tokens": 202800, "max_output_tokens": 202800, - "max_tokens": 202800, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73341,7 +70529,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 4.05e-06, "source": "https://inference.baseten.co/v1/models", @@ -73359,7 +70546,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -73376,7 +70562,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -73393,7 +70578,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inference.baseten.co/v1/models", @@ -73410,7 +70594,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.6e-07, "source": "https://inference.baseten.co/v1/models", @@ -73428,7 +70611,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.48e-06, "source": "https://inference.baseten.co/v1/models", @@ -73445,7 +70627,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 3.96e-06, "source": "https://inference.baseten.co/v1/models", @@ -73462,7 +70643,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 6.6e-06, "source": "https://inference.baseten.co/v1/models", @@ -73482,7 +70662,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, @@ -73503,7 +70682,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, @@ -73524,7 +70702,6 @@ "litellm_provider": "xai", "max_input_tokens": 256000, "max_output_tokens": 256000, - "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, @@ -73545,7 +70722,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.2e-06, @@ -73563,7 +70739,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 196608, "max_output_tokens": 196608, - "max_tokens": 196608, "mode": "chat", "output_cost_per_token_priority": 1.2e-06, "source": "https://api.fireworks.ai/v1/serverless/models", @@ -73629,7 +70804,6 @@ "litellm_provider": "vertex_ai-zai_models", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 4.4e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -73647,7 +70821,6 @@ "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "output_cost_per_token_batches": 3.6e-07, @@ -73666,7 +70839,6 @@ "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.4, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo", @@ -73682,7 +70854,6 @@ "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.1, "output_cost_per_second_1080p": 0.12, @@ -73699,7 +70870,6 @@ "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, - "max_tokens": 1024, "mode": "video_generation", "output_cost_per_second": 0.5, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo", @@ -73746,7 +70916,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73779,7 +70948,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73812,7 +70980,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73845,7 +71012,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73883,7 +71049,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73916,7 +71081,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "thinking_always_on": true, @@ -73952,7 +71116,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -73991,7 +71154,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74031,7 +71193,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74069,7 +71230,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74108,7 +71268,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "supports_assistant_prefill": false, "supports_computer_use": true, @@ -74141,7 +71300,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "supports_mid_conversation_system": true, @@ -74179,7 +71337,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -74199,7 +71356,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -74219,7 +71375,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "thinking_always_on": true, "supports_function_calling": true, @@ -74241,7 +71396,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8192, - "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 5.4e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-r1.html", @@ -74254,7 +71408,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-06, "source": "https://aws.amazon.com/bedrock/pricing/", @@ -74267,7 +71420,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.5e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", @@ -74303,7 +71455,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -74341,7 +71492,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "supports_mid_conversation_system": true, @@ -74387,7 +71537,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74429,7 +71578,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -74466,7 +71614,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -74503,7 +71650,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -74544,7 +71690,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74591,7 +71736,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "search_context_cost_per_query": { @@ -74633,7 +71777,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74765,7 +71908,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, "supports_forced_tool_use": false, @@ -74811,7 +71953,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.1e-05, "search_context_cost_per_query": { @@ -74856,7 +71997,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -74896,7 +72036,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -74929,7 +72068,6 @@ "litellm_provider": "prism", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 6.3e-07, "source": "https://prisminference.com/pricing", @@ -74954,7 +72092,6 @@ "litellm_provider": "prism", "max_input_tokens": 1000000, "max_output_tokens": 384000, - "max_tokens": 384000, "mode": "chat", "output_cost_per_token": 2.1e-07, "source": "https://prisminference.com/pricing", @@ -74987,7 +72124,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_flex": 3e-06, @@ -75013,7 +72149,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6.6e-06, "output_cost_per_token_flex": 3.3e-06, @@ -75031,7 +72166,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -75046,10 +72180,14 @@ "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": 6e-05, + "cache_creation_input_token_cost_above_12hr_batches": 2.5e-06, "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, "cache_creation_input_token_cost_above_12hr_priority": 1e-05, + "cache_creation_input_token_cost_above_12hr_ultrafast": 3e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, @@ -75082,7 +72220,6 @@ "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -75150,7 +72287,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, @@ -75196,7 +72332,6 @@ "litellm_provider": "bedrock_mantle", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "responses", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -75244,7 +72379,6 @@ "supports_bedrock_converse_tool_result_images": false, "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.1e-05, "output_cost_per_token_above_272k_tokens": 1.65e-05, @@ -75279,7 +72413,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 9e-06, "output_cost_per_token": 9e-06, @@ -75293,7 +72426,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_output_tokens": 16384, - "max_tokens": 16384, "mode": "audio_speech", "output_cost_per_audio_token": 6e-06, "output_cost_per_token": 6e-06, @@ -75310,7 +72442,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 524288, "max_output_tokens": 524288, - "max_tokens": 524288, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -75328,7 +72459,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 262144, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.5e-06, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/kimi/", @@ -75361,7 +72491,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.28e-06, "output_cost_per_token_batches": 2.64e-06, @@ -75394,7 +72523,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.808e-06, "output_cost_per_token_batches": 2.904e-06, @@ -75417,7 +72545,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "realtime", "output_cost_per_audio_token": 1.2e-05, "output_cost_per_token": 2.75e-06, @@ -75432,7 +72559,6 @@ "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "chat", "output_cost_per_token": 0.0, "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/model-router/", @@ -75452,7 +72578,6 @@ "litellm_provider": "gemini", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -75497,7 +72622,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -75543,7 +72667,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 32768, - "max_tokens": 32768, "mode": "image_generation", "output_cost_per_image": 0.0336, "output_cost_per_image_token": 3e-05, @@ -75587,7 +72710,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -75615,7 +72737,6 @@ "token_count_tokenizer": "kimi_k3", "max_input_tokens": 1000000, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, @@ -75648,7 +72769,6 @@ "litellm_provider": "anthropic", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -75702,7 +72822,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -75749,7 +72868,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -75794,7 +72912,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "search_context_cost_per_query": { @@ -75837,7 +72954,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "supports_adaptive_thinking": true, "supports_mid_conversation_system": true, @@ -75888,7 +73004,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -75935,7 +73050,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "search_context_cost_per_query": { @@ -75977,7 +73091,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -76019,7 +73132,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -76065,7 +73177,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -76117,7 +73228,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "search_context_cost_per_query": { @@ -76164,7 +73274,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -76223,7 +73332,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6e-07, "supports_forced_tool_use": true, @@ -76274,7 +73382,6 @@ "supports_tool_search": true, "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5.5e-07, "search_context_cost_per_query": { @@ -76324,7 +73431,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -76369,7 +73475,6 @@ "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -76408,7 +73513,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 261120, "max_output_tokens": 98304, - "max_tokens": 98304, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", @@ -76419,7 +73523,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 261120, "max_output_tokens": 98304, - "max_tokens": 98304, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", @@ -76430,7 +73533,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 261120, "max_output_tokens": 98304, - "max_tokens": 98304, "mode": "chat", "output_cost_per_token": 7.5e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-twelvelabs-pegasus-v1-5.html", @@ -76439,9 +73541,10 @@ "azure_ai/Microsoft-Decision-1": { "input_cost_per_token": 4.2e-08, "litellm_provider": "azure_ai", + "max_input_tokens": 32768, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://prices.azure.com/api/retail/prices?$filter=productName eq 'Microsoft Decision Models'" + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure" }, "gpt-rosalind-discovery": { "cache_read_input_token_cost": 5e-07, @@ -76465,7 +73568,6 @@ "input_cost_per_token": 8e-08, "litellm_provider": "azure_ai", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048.0, @@ -76485,7 +73587,6 @@ "input_cost_per_token": 1.2e-07, "litellm_provider": "azure_ai", "max_input_tokens": 128000, - "max_tokens": 128000, "mode": "embedding", "output_cost_per_token": 0.0, "output_vector_size": 2048.0, @@ -76554,7 +73655,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, @@ -76600,7 +73700,6 @@ "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, @@ -76650,7 +73749,6 @@ "litellm_provider": "gemini", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_reasoning_token": 1e-05, "output_cost_per_token": 1e-05, @@ -76709,7 +73807,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_reasoning_token": 1e-05, "output_cost_per_token": 1e-05, @@ -76767,7 +73864,6 @@ "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_reasoning_token": 2e-05, "output_cost_per_token": 2e-05, @@ -76815,7 +73911,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 65536, - "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -76828,7 +73923,6 @@ "supports_vision": true }, "openinfer/@oi/GLM-5.3-Flash": { - "max_tokens": 393216, "max_input_tokens": 393216, "max_output_tokens": 393216, "input_cost_per_token": 1.95e-07, @@ -76840,7 +73934,6 @@ "source": "https://api.openinfer.ai/v1/models" }, "openinfer/@oi/Qwen3.5-27B": { - "max_tokens": 32768, "max_input_tokens": 32768, "max_output_tokens": 32768, "input_cost_per_token": 1.95e-07, @@ -76852,7 +73945,6 @@ "source": "https://api.openinfer.ai/v1/models" }, "openinfer/@oi/Qwen3.5-9B": { - "max_tokens": 100000, "max_input_tokens": 100000, "max_output_tokens": 100000, "input_cost_per_token": 5e-06, @@ -76931,7 +74023,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "together_ai", "max_input_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 262144, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://api.together.ai/v1/models", @@ -76954,7 +74046,6 @@ "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 3.311e-06, "output_cost_per_token_batches": 1.6555e-06, @@ -76976,7 +74067,6 @@ "litellm_provider": "vertex_ai-deepseek_models", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -77000,7 +74090,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 4e-05, "prompt_cache_min_tokens": 512.0, @@ -77030,7 +74119,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.25e-06, "source": "https://inference.baseten.co/v1/models", @@ -77048,7 +74136,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 5e-07, "source": "https://inference.baseten.co/v1/models", @@ -77064,7 +74151,7 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "wandb", "max_input_tokens": 1049000, - "max_tokens": 943718, + "max_output_tokens": 943718, "mode": "chat", "output_cost_per_token": 7.5e-07, "source": "https://openrouter.ai/api/v1/models/nvidia/nemotron-3.5-super-vl/endpoints", @@ -77080,7 +74167,7 @@ "input_cost_per_token": 4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -77094,7 +74181,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, - "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -77108,7 +74194,7 @@ "input_cost_per_token": 2.5e-07, "litellm_provider": "nebius", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://tokenfactory.nebius.com/models/catalog/text2text/reflection%2Fbeam", @@ -77121,7 +74207,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", @@ -77138,7 +74223,6 @@ "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 262144, - "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 2.4e-06, "source": "https://inference.baseten.co/v1/models", @@ -77157,7 +74241,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, @@ -77177,7 +74260,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -77199,7 +74281,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 272000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "responses", "output_cost_per_token": 1.25e-06, "source": "https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml", @@ -77221,7 +74302,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -77243,7 +74323,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -77265,7 +74344,6 @@ "litellm_provider": "github_copilot", "max_input_tokens": 500000, "max_output_tokens": 500000, - "max_tokens": 500000, "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, @@ -77284,7 +74362,7 @@ "input_cost_per_token": 3e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, - "max_tokens": 1048576, + "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models", @@ -77321,7 +74399,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, - "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 9e-07, @@ -77357,7 +74434,6 @@ "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 393216, - "max_tokens": 393216, "mode": "chat", "output_cost_per_token": 1.8e-06, "output_cost_per_token_priority": 2.25e-06, @@ -77406,7 +74482,6 @@ "litellm_provider": "azure_ai", "max_input_tokens": 256000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -77436,7 +74511,6 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07, - "max_tokens": 500000, "max_output_tokens": 500000, "max_input_tokens": 500000, "supports_function_calling": true, @@ -77474,7 +74548,6 @@ "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, "cache_creation_input_token_cost": 1.25e-07, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 922000, "supports_function_calling": true, @@ -77514,7 +74587,6 @@ "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 2e-08, "cache_creation_input_token_cost": 2.5e-07, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 922000, "supports_function_calling": true, @@ -77554,7 +74626,6 @@ "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, "cache_creation_input_token_cost": 1.25e-07, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77593,7 +74664,6 @@ "input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05, "cache_read_input_token_cost": 1e-07, - "max_tokens": 65536, "max_output_tokens": 65536, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77610,7 +74680,6 @@ "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 3e-08, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77627,7 +74696,6 @@ "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77644,7 +74712,6 @@ "input_cost_per_token": 1.4e-06, "output_cost_per_token": 4.4e-06, "cache_read_input_token_cost": 2.6e-07, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77662,7 +74729,6 @@ "output_cost_per_token": 1e-05, "cache_read_input_token_cost": 2.0000000000000002e-07, "cache_creation_input_token_cost": 2e-06, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77680,7 +74746,6 @@ "output_cost_per_token": 1.5e-05, "cache_read_input_token_cost": 3e-07, "cache_creation_input_token_cost": 3e-06, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77697,7 +74762,6 @@ "input_cost_per_token": 9.499999999999999e-07, "output_cost_per_token": 4e-06, "cache_read_input_token_cost": 1.9e-07, - "max_tokens": 262144, "max_output_tokens": 262144, "max_input_tokens": 262144, "supports_function_calling": true, @@ -77714,7 +74778,6 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77731,7 +74794,6 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-09, - "max_tokens": 384000, "max_output_tokens": 384000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77748,7 +74810,6 @@ "input_cost_per_token": 1.32e-06, "output_cost_per_token": 3.96e-06, "cache_read_input_token_cost": 4.4e-08, - "max_tokens": 384000, "max_output_tokens": 384000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77765,7 +74826,6 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, "cache_read_input_token_cost": 2.8e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77782,7 +74842,6 @@ "input_cost_per_token": 4.35e-07, "output_cost_per_token": 8.7e-07, "cache_read_input_token_cost": 3.625e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77799,7 +74858,6 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, "cache_read_input_token_cost": 2.8e-09, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77816,7 +74874,6 @@ "input_cost_per_token": 4.35e-07, "output_cost_per_token": 8.7e-07, "cache_read_input_token_cost": 3.625e-09, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77833,7 +74890,6 @@ "input_cost_per_token": 1.5e-07, "output_cost_per_token": 6e-07, "cache_read_input_token_cost": 1.5e-08, - "max_tokens": 512000, "max_output_tokens": 512000, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77850,7 +74906,6 @@ "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "cache_read_input_token_cost": 6e-08, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77867,7 +74922,6 @@ "input_cost_per_token": 1.0000000000000001e-07, "output_cost_per_token": 2.0000000000000002e-07, "cache_read_input_token_cost": 2e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77884,7 +74938,6 @@ "input_cost_per_token": 1.0000000000000001e-07, "output_cost_per_token": 2.0000000000000002e-07, "cache_read_input_token_cost": 2e-09, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -77902,7 +74955,6 @@ "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 2.5e-07, "cache_creation_input_token_cost": 2.5e-06, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77920,7 +74972,6 @@ "output_cost_per_token": 4.6999999999999995e-07, "cache_read_input_token_cost": 1.6e-08, "cache_creation_input_token_cost": 2.0000000000000002e-07, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77938,7 +74989,6 @@ "output_cost_per_token": 1.6000000000000001e-06, "cache_read_input_token_cost": 4e-08, "cache_creation_input_token_cost": 5e-07, - "max_tokens": 65536, "max_output_tokens": 65536, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -77977,7 +75027,6 @@ "input_cost_per_token": 8.34e-07, "output_cost_per_token": 2.501e-06, "cache_read_input_token_cost": 4.2000000000000006e-08, - "max_tokens": 64000, "max_output_tokens": 64000, "max_input_tokens": 1024000, "supports_function_calling": true, @@ -77994,7 +75043,6 @@ "input_cost_per_token": 1.4e-07, "output_cost_per_token": 5.8e-07, "cache_read_input_token_cost": 3.5e-08, - "max_tokens": 128000, "max_output_tokens": 128000, "max_input_tokens": 192000, "supports_function_calling": true, @@ -78011,7 +75059,6 @@ "input_cost_per_token": 0, "output_cost_per_token": 0, "cache_read_input_token_cost": 0, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -78028,7 +75075,6 @@ "input_cost_per_token": 1e-06, "output_cost_per_token": 2.7e-06, "cache_read_input_token_cost": 1.0000000000000001e-07, - "max_tokens": 65536, "max_output_tokens": 65536, "max_input_tokens": 1000000, "supports_function_calling": true, @@ -78045,7 +75091,6 @@ "input_cost_per_token": 0, "output_cost_per_token": 0, "cache_read_input_token_cost": 0, - "max_tokens": 131072, "max_output_tokens": 131072, "max_input_tokens": 1048576, "supports_function_calling": true, @@ -78066,7 +75111,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -78111,7 +75155,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-05, "output_cost_per_token_batches": 2.5e-05, @@ -78155,7 +75198,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, @@ -78188,7 +75230,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, @@ -78238,7 +75279,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78275,7 +75315,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78318,7 +75357,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78362,7 +75400,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78407,7 +75444,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2.5e-05, "output_cost_per_token_batches": 1.25e-05, @@ -78452,7 +75488,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 2e-05, "output_cost_per_token_batches": 1e-05, @@ -78497,7 +75532,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 64000, - "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -78532,7 +75566,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, @@ -78575,7 +75608,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -78619,7 +75651,6 @@ "litellm_provider": "bedrock", "max_input_tokens": 1000000, "max_output_tokens": 128000, - "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, @@ -78652,5 +75683,327 @@ "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/about-claude/pricing#claude-platform-on-aws-pricing", "supports_web_search": true + }, + "gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_12hr": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_above_12hr_batches": 1.25e-06, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-06, + "cache_creation_input_token_cost_above_12hr_priority": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-06, + "cache_creation_input_token_cost_batches": 6.25e-07, + "cache_creation_input_token_cost_flex": 6.25e-07, + "cache_creation_input_token_cost_priority": 2.5e-06, + "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 5e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 2e-07, + "cache_read_input_token_cost_batches": 2.5e-08, + "cache_read_input_token_cost_flex": 2.5e-08, + "cache_read_input_token_cost_priority": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_272k_tokens": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_flex": 1e-06, + "input_cost_per_token_above_272k_tokens_priority": 4e-06, + "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_priority": 2e-06, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "output_cost_per_token_above_272k_tokens": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-06, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-06, + "output_cost_per_token_above_272k_tokens_priority": 1.5e-05, + "output_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_flex": 2.5e-06, + "output_cost_per_token_priority": 1e-05, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01}, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_tool_search": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, + "chatgpt/gpt-6.1-terra": { + "litellm_provider": "chatgpt", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "responses", + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "global.openai.gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, + "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_272k_tokens": 2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-06, + "output_cost_per_token_above_272k_tokens": 7.5e-06, + "source": "https://developers.openai.com/api/docs/models/gpt-6.1-terra", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_bedrock_converse_tool_result_images": false, + "supports_bedrock_runtime_chat_completions_response_format": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "us.openai.gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.75e-06, + "cache_read_input_token_cost": 5.5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-07, + "input_cost_per_token": 1.1e-06, + "input_cost_per_token_above_272k_tokens": 2.2e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.5e-06, + "output_cost_per_token_above_272k_tokens": 8.25e-06, + "source": "https://developers.openai.com/api/docs/models/gpt-6.1-terra", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_bedrock_converse_tool_result_images": false, + "supports_bedrock_runtime_chat_completions_response_format": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true + }, + "baseten/decision-models/baseten-elephant-9b-latest": { + "input_cost_per_token": 0.0, + "litellm_provider": "baseten", + "max_input_tokens": 16384, + "mode": "decisions", + "output_cost_per_token": 0.0, + "source": "https://inference.baseten.co/v1/models", + "supports_vision": true + }, + "baseten/decision-models/baseten-mammoth-27b-latest": { + "input_cost_per_token": 0.0, + "litellm_provider": "baseten", + "max_input_tokens": 16384, + "mode": "decisions", + "output_cost_per_token": 0.0, + "source": "https://inference.baseten.co/v1/models", + "supports_vision": true + }, + "github_copilot/gpt-6.1-terra": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, + "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_above_272k_tokens": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_272k_tokens": 2e-06, + "litellm_provider": "github_copilot", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 5e-06, + "output_cost_per_token_above_272k_tokens": 7.5e-06, + "source": "https://github.blog/changelog/2026-10-26-gpt-6-1-terra-in-github-copilot", + "supported_endpoints": [ + "/v1/responses" + ], + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "refuseless/glm-5.3-abliterated": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "refuseless", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2.25e-06, + "source": "https://refuseless.com/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ] + }, + "refuseless/glm-5.3-flash-abliterated": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "refuseless", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8e-07, + "source": "https://refuseless.com/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ] + }, + "refuseless/deepseek-v4.1-flash-abliterated": { + "cache_read_input_token_cost": 4.5e-08, + "input_cost_per_token": 2.2e-07, + "litellm_provider": "refuseless", + "max_input_tokens": 1048576, + "max_output_tokens": 130000, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "source": "https://refuseless.com/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ] + }, + "refuseless/deepseek-v4.1-flash-fast": { + "cache_read_input_token_cost": 3e-09, + "input_cost_per_token": 5e-08, + "litellm_provider": "refuseless", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://refuseless.com/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ] + }, + "refuseless/minimax-m3.1-flash": { + "cache_read_input_token_cost": 2e-09, + "input_cost_per_token": 5e-08, + "litellm_provider": "refuseless", + "max_input_tokens": 1048576, + "max_output_tokens": 512000, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://refuseless.com/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ] } } diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index 890fa0fd773..27681d9116c 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -3,14 +3,14 @@ name = "litellm-core" version = "1.105.0" description = "Library to easily interface with LLM API providers" readme = "README.md" -requires-python = ">=3.10, <3.15" +requires-python = ">=3.10, <3.16" license = "MIT" license-files = ["LICENSE"] authors = [ { name = "BerriAI" }, ] dependencies = [ - "fastuuid>=0.14.0,<1.0", + "fastuuid>=0.14.0,<1.0; python_version < '3.15'", "filelock>=3.16.1,<4.0", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", @@ -20,14 +20,16 @@ dependencies = [ "packaging>=24.0", "importlib-metadata>=8.0.0,<9.0", "tiktoken>=0.8.0,<1.0; python_version < '3.14'", - "tiktoken>=0.12.0,<1.0; python_version >= '3.14'", + "tiktoken>=0.12.0,<1.0; python_version == '3.14'", + "tiktoken>=0.14.0,<1.0; python_version >= '3.15'", "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", "anyio>=4.14.0,<5.0", "async-timeout>=4.0.3,<6.0; python_version < '3.11'", "pydantic>=2.11.0,<3.0.0; python_version < '3.14'", - "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", + "pydantic>=2.12.0,<3.0.0; python_version == '3.14'", + "pydantic>=2.14.0,<3.0.0; python_version >= '3.15'", "pydantic-settings>=2.14.1,<3.0", "jsonschema>=4.0.0,<5.0", "certifi>=2026.10.19", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index d65da3f41fd..e62b74aa87d 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -33,6 +33,23 @@ } }, "providers": { + "refuseless": { + "display_name": "Refuseless (`refuseless`)", + "url": "https://docs.litellm.ai/docs/providers/refuseless", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "a2a": { "display_name": "A2A (Agent-to-Agent) (`a2a`)", "url": "https://docs.litellm.ai/docs/providers/a2a", diff --git a/pyproject.toml b/pyproject.toml index 817c50df90b..fc860a8fdf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "litellm" version = "1.108.0" description = "Library to easily interface with LLM API providers" readme = "README.md" -requires-python = ">=3.10, <3.15" +requires-python = ">=3.10, <3.16" license = "MIT" license-files = ["LICENSE"] authors = [ @@ -14,7 +14,7 @@ dependencies = [ # deps. Reproducibility for our Docker/CI comes from `uv.lock`. # When changing a floor, verify it installs + imports on every supported # Python with: `uv pip install --resolution=lowest-direct .` - "fastuuid>=0.14.0,<1.0", + "fastuuid>=0.14.0,<1.0; python_version < '3.15'", "filelock>=3.16.1,<4.0", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", @@ -23,7 +23,8 @@ dependencies = [ "packaging>=24.0", "importlib-metadata>=8.0.0,<9.0", "tiktoken>=0.8.0,<1.0; python_version < '3.14'", - "tiktoken>=0.12.0,<1.0; python_version >= '3.14'", + "tiktoken>=0.12.0,<1.0; python_version == '3.14'", + "tiktoken>=0.14.0,<1.0; python_version >= '3.15'", "tokenizers>=0.21.0,<1.0", "huggingface-hub>=0.34.0,<2.0", "click>=8.0.0,<9.0", @@ -32,7 +33,8 @@ dependencies = [ "anyio>=4.14.0,<5.0", "async-timeout>=4.0.3,<6.0; python_version < '3.11'", "pydantic>=2.11.0,<3.0.0; python_version < '3.14'", - "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", + "pydantic>=2.12.0,<3.0.0; python_version == '3.14'", + "pydantic>=2.14.0,<3.0.0; python_version >= '3.15'", "pydantic-settings>=2.14.1,<3.0", "jsonschema>=4.0.0,<5.0", "boto3>=1.43.117,<2.0", @@ -55,17 +57,20 @@ Documentation = "https://docs.litellm.ai" proxy = [ "gunicorn>=23.0.0,<24.0", "uvicorn>=0.33.0,<1.0", - "granian>=2.7.4,<3.0", + "granian>=2.7.4,<3.0; python_version < '3.15'", + "granian>=2.8.0,<3.0; python_version >= '3.15'", "uvloop>=0.22.1,<1.0; sys_platform != 'win32'", "fastapi>=0.136.3,<1.0", "starlette>=1.0.1,<2.0", "backoff>=2.2.1,<3.0", "pyyaml>=6.0.3,<7.0", "rq>=2.7.0,<3.0", - "orjson>=3.11.6,<4.0", + "orjson>=3.11.6,<4.0; python_version < '3.15'", + "orjson>=3.12.0,<4.0; python_version >= '3.15'", # redis-py's C response parser. It arrives with redis (via rq) either way; naming # it here is what makes redis-py select _HiredisParser instead of the Python one. - "hiredis>=3.0.0,<4.0", + "hiredis>=3.0.0,<4.0; python_version < '3.15'", + "hiredis>=3.5.0,<4.0; python_version >= '3.15'", "apscheduler>=3.11.2,<4.0", "fastapi-sso>=0.19.0,<1.0", "PyJWT>=2.13.0,<3.0", @@ -210,8 +215,8 @@ dev = [ "tomli==2.4.1; python_version < '3.11'", "pytest-mock==3.15.1", "pytest-asyncio==1.3.0", - "psycopg==3.3.3", - "psycopg-binary==3.3.3", + "psycopg==3.3.6", + "psycopg-binary==3.3.6", "pytest-xdist==3.8.0", "requests-mock==1.12.1", "responses==0.26.0", @@ -372,21 +377,8 @@ required-version = ">=0.10.9" exclude-newer = "3 days" [tool.uv.exclude-newer-package] -# TODO: ryan, remove after 2026-10-23, once certifi 2026.10.19 (drops the compromised eMudhra root) clears the 3-day window -certifi = "2026-10-20T03:00:00Z" -# TODO(shrey): remove after 2026-10-25, once grpcio 1.84.1 (drops the compromised eMudhra roots from roots.pem) clears the 3-day window -grpcio = "2026-10-22T15:00:00Z" -# TODO(shrey): remove after 2026-10-24, once boto3/botocore 1.43.117 (drops the compromised eMudhra roots from cacert.pem) clear the 3-day window -boto3 = "2026-10-21T19:31:00Z" -botocore = "2026-10-21T19:31:00Z" # TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window brotli = "2026-10-22T14:01:00Z" -# TODO: ryan, remove after 2026-10-24, once mlflow 3.17.2 (GHSA-wqmh-qccw-pqwf) clears the 3-day window -mlflow = "2026-10-21T03:00:00Z" -mlflow-skinny = "2026-10-21T03:00:00Z" -mlflow-tracing = "2026-10-21T03:00:00Z" -# TODO: ryan, remove after 2026-10-25, once pypdf 6.21.2 (GHSA-6c7f-9265-g8fw) clears the 3-day window -pypdf = "2026-10-21T17:00:00Z" [tool.uv.sources] litellm-proxy-extras = { workspace = true } diff --git a/ruff-strict.toml b/ruff-strict.toml index 28102da3ccd..398cca15df2 100644 --- a/ruff-strict.toml +++ b/ruff-strict.toml @@ -2,7 +2,7 @@ extend = "ruff.toml" [lint] preview = true -select = ["ANN", "ASYNC230", "B004", "B006", "B008", "B009", "B010", "B018", "B019", "B021", "B026", "B033", "BLE", "C401", "C404", "C405", "C408", "C414", "C419", "C901", "D419", "DTZ001", "DTZ003", "DTZ005", "DTZ006", "DTZ007", "DTZ011", "EXE001", "EXE002", "F401", "FURB136", "FURB168", "FURB188", "I001", "LOG015", "N999", "PERF102", "PERF401", "PERF402", "PERF403", "PIE790", "PIE800", "PIE804", "PIE810", "PLC0206", "PLC0208", "PLC0414", "PLR0124", "PLR0206", "PLR0402", "PLR1704", "PLR1711", "PLR1714", "PLR1730", "PLR2044", "PLW0127", "PLW0133", "PLW0602", "PLW0603", "PLW1508", "PLW1510", "PYI030", "PYI036", "PYI041", "PYI064", "RET501", "RET504", "RUF010", "RUF012", "RUF015", "RUF019", "RUF022", "RUF023", "RUF046", "RUF051", "RUF059", "RUF100", "S110", "S112", "SIM101", "SIM102", "SIM103", "SIM113", "SIM114", "SIM115", "SIM117", "SIM118", "SIM201", "SIM210", "SIM211", "SIM222", "SIM401", "TC004", "TC005", "TID251", "TRY002", "TRY004", "TRY201", "TRY203", "TRY300", "UP006", "UP007", "UP008", "UP012", "UP018", "UP024", "UP028", "UP031", "UP032", "UP034", "UP035", "UP036", "UP037", "UP045"] +select = ["ANN", "ASYNC230", "B004", "B006", "B008", "B009", "B010", "B018", "B019", "B021", "B026", "B033", "BLE", "C401", "C404", "C405", "C408", "C414", "C419", "C901", "D202", "D403", "D417", "D419", "DTZ001", "DTZ003", "DTZ005", "DTZ006", "DTZ007", "DTZ011", "EXE001", "EXE002", "F401", "FURB136", "FURB168", "FURB188", "I001", "LOG015", "N999", "PERF102", "PERF401", "PERF402", "PERF403", "PIE790", "PIE800", "PIE804", "PIE810", "PLC0206", "PLC0208", "PLC0414", "PLR0124", "PLR0206", "PLR0402", "PLR1704", "PLR1711", "PLR1714", "PLR1730", "PLR2044", "PLW0127", "PLW0133", "PLW0602", "PLW0603", "PLW1508", "PLW1510", "PYI030", "PYI036", "PYI041", "PYI064", "RET501", "RET504", "RUF010", "RUF012", "RUF015", "RUF019", "RUF022", "RUF023", "RUF046", "RUF051", "RUF059", "RUF100", "S110", "S112", "SIM101", "SIM102", "SIM103", "SIM113", "SIM114", "SIM115", "SIM117", "SIM118", "SIM201", "SIM210", "SIM211", "SIM222", "SIM401", "TC004", "TC005", "TID251", "TRY002", "TRY004", "TRY201", "TRY203", "TRY300", "UP006", "UP007", "UP008", "UP012", "UP018", "UP024", "UP028", "UP031", "UP032", "UP034", "UP035", "UP036", "UP037", "UP045"] extend-select = [] # Overrides the inherited list: rules this gate enforces itself must NOT be external here, # so this config's RUF100 flags their stale `# noqa` directives. What remains external is @@ -39,6 +39,9 @@ external = [ [lint.mccabe] max-complexity = 15 +[lint.pydocstyle] +convention = "google" + [lint.pylint] max-args = 5 diff --git a/ruff.toml b/ruff.toml index 93396a723e7..715e5551fad 100644 --- a/ruff.toml +++ b/ruff.toml @@ -20,7 +20,7 @@ lint.external = [ # Family entries whose every strict rule graduated into extend-select above (FURB), and # standalone graduated codes (I001, RUF010, RUF022, RUF023, RUF051), are dropped so this # config's RUF100 polices their directives itself. - "ANN", "ASYNC230", "B", "C", "D419", "DTZ", "EXE", "LOG015", "N999", "PERF", + "ANN", "ASYNC230", "B", "C", "D202", "D403", "D417", "D419", "DTZ", "EXE", "LOG015", "N999", "PERF", "PIE", "PL", "PYI", "RET", "RUF012", "RUF015", "RUF019", "RUF046", "RUF059", "S110", "S112", "SIM", "TC", "TID251", "TRY", "UP", # Enforced by upstream litellm's ruff config, but not run in this repo's CI diff --git a/scripts/sync_together_ai_models.py b/scripts/sync_together_ai_models.py deleted file mode 100644 index 12b128890f1..00000000000 --- a/scripts/sync_together_ai_models.py +++ /dev/null @@ -1,547 +0,0 @@ -"""Sync the together_ai entries of model_prices_and_context_window.json with Together's live serverless catalog. - -Pulls ``GET https://api.together.ai/v1/models?serverless`` plus the deprecations doc, maps API fields onto -registry fields, merges the reviewed capability rules below for everything the API cannot express, and diffs -the result against the registry. Dry run (the default) prints the diff summary and the generated PR body; -``--write`` applies the changes to the root cost map and its ``litellm/`` backup copy. - -Policy highlights: -- Prices arrive per 1M tokens with float artifacts and are normalized to clean per-token values. -- A registry entry absent from the serverless catalog is marked with ``deprecation_date`` from the docs - deprecation table, never deleted; absences with no docs date are surfaced for a human call. -- Availability comes from the API: a model the docs list as removed but the API still serves stays live, - with the conflict surfaced as a warning. -- Manually curated values the API cannot express (``metadata.successor``, ``max_output_tokens`` on existing - entries, capability flags no rule covers) are never overwritten; conflicts are surfaced instead. -""" - -import argparse -import json -import os -import re -import sys -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field -from pathlib import Path -from types import MappingProxyType -from typing import Final - -import httpx -from pydantic import BaseModel, TypeAdapter, ValidationError - -MODELS_URL: Final = "https://api.together.ai/v1/models?serverless" -DEPRECATIONS_URL: Final = "https://docs.together.ai/docs/deprecations.md" -PROVIDER: Final = "together_ai" -PREFIX: Final = "together_ai/" -SOURCE_URL: Final = "https://docs.together.ai/docs/serverless-models" -COST_MAP_RELPATHS: Final = ( - "model_prices_and_context_window.json", - "litellm/model_prices_and_context_window_backup.json", -) - -TYPE_TO_MODE: Final = MappingProxyType({"chat": "chat", "embedding": "embedding", "moderation": "chat"}) - - -class SyncError(RuntimeError): - pass - - -class CatalogPricing(BaseModel): - input: float - output: float - cached_input: float | None = None - - -class CatalogModel(BaseModel): - id: str - type: str - context_length: int | None = None - pricing: CatalogPricing - - -CATALOG_ADAPTER: Final = TypeAdapter(list[CatalogModel]) - -RegistryEntry = dict[str, object] -CostMap = dict[str, object] - - -@dataclass(frozen=True, slots=True) -class CapabilityRule: - model_id: str - fields: Mapping[str, bool | int] - provenance: str - - -def _rule(model_id: str, provenance: str, **fields: bool | int) -> CapabilityRule: - return CapabilityRule(model_id=model_id, fields=MappingProxyType(dict(fields)), provenance=provenance) - - -_TOOLS: Final = MappingProxyType( - { - "supports_function_calling": True, - "supports_parallel_function_calling": True, - "supports_response_schema": True, - "supports_tool_choice": True, - } -) - -CAPABILITY_RULES: Final = ( - _rule( - "MiniMaxAI/MiniMax-M3", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/minimax-m3", - **_TOOLS, - supports_reasoning=True, - supports_vision=True, - ), - _rule("Prism-ML/Ternary-Bonsai-27B", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule( - "Qwen/Qwen3.5-9B", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/qwen3-5-9b", - **_TOOLS, - supports_reasoning=True, - supports_vision=True, - ), - _rule( - "Qwen/Qwen3.6-Plus", - "reviewed for the LIT-5968 backfill; hybrid reasoning model without a documented tools contract", - supports_reasoning=True, - ), - _rule("Qwen/Qwen3.7-Max", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule("Qwen/Qwen3.7-Plus", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule("Qwen/Qwen3.8-2.4T-A95B", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule("arize-ai/qwen-2-1.5b-instruct", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule( - "deepseek-ai/DeepSeek-V4-Flash-0731", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/deepseek-v4-flash", - **_TOOLS, - ), - _rule( - "deepseek-ai/DeepSeek-V4-Pro", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/deepseek-v4-pro", - **_TOOLS, - supports_reasoning=True, - ), - _rule( - "deepseek-ai/DeepSeek-V4-Pro-0813", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/deepseek-v4-pro", - **_TOOLS, - ), - _rule("google/gemma-3n-E4B-it", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule( - "google/gemma-4-31B-it", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/gemma-4-31b-it", - **_TOOLS, - supports_vision=True, - ), - _rule( - "intfloat/multilingual-e5-large-instruct", - "embedding dims per https://huggingface.co/intfloat/multilingual-e5-large-instruct", - output_vector_size=1024, - ), - _rule( - "meta-llama/Llama-3.3-70B-Instruct-Turbo", - "reviewed for the LIT-5968 backfill against https://docs.together.ai/docs/function-calling", - **_TOOLS, - ), - _rule( - "meta-llama/Llama-Guard-4-12B", - "moderation classifier with a chat-shaped API; no tools per the LIT-5968 backfill review", - ), - _rule("meta-models/Muse-Glimmer-30B", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule( - "moonshotai/Kimi-K2.7-Code", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/kimi-k2-7-code", - **_TOOLS, - supports_vision=True, - ), - _rule( - "moonshotai/Kimi-K3", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/kimi-k3", - **_TOOLS, - supports_reasoning=True, - supports_vision=True, - ), - _rule( - "nvidia/nemotron-3-ultra-550b-a55b", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/nemotron-3-ultra", - **_TOOLS, - supports_reasoning=True, - ), - _rule( - "openai/gpt-oss-120b", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/gpt-oss-120b", - **_TOOLS, - supports_reasoning=True, - ), - _rule( - "openai/gpt-oss-20b", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/gpt-oss-20b", - **_TOOLS, - ), - _rule("pearl-ai/gemma-4-31b-it", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule( - "thinkingmachines/Inkling", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/inkling", - **_TOOLS, - ), - _rule("thinkingmachines/Inkling-Small", "reviewed for the LIT-5968 backfill; no tool or vision support documented"), - _rule( - "zai-org/GLM-5.2", - "reviewed for the LIT-5968 backfill against https://www.together.ai/models/glm-5-2;" - " 128K output ceiling per https://docs.z.ai/guides/llm/glm-5.2", - **_TOOLS, - supports_reasoning=True, - max_output_tokens=128000, - max_tokens=128000, - ), - _rule( - "zai-org/GLM-5.3-Flash", - "reviewed for LIT-6489 against https://www.together.ai/models/glm-5-3-flash;" - " 128K output ceiling per https://docs.z.ai/guides/llm/glm-5.3", - **_TOOLS, - supports_reasoning=True, - supports_vision=True, - max_output_tokens=128000, - max_tokens=128000, - ), -) - -RULES_BY_ID: Final = MappingProxyType({rule.model_id: rule for rule in CAPABILITY_RULES}) - - -@dataclass(frozen=True, slots=True) -class DeprecationDoc: - removal_dates: Mapping[str, str] - redirects: Mapping[str, str] - - -_REDIRECT_ROW: Final = re.compile(r"^\|\s*`([^`]+)`\s*\|\s*`([^`]+)`\s*\|") -_REMOVAL_ROW: Final = re.compile(r"^\|\s*(\d{4}-\d{2}-\d{2})\s*\|\s*`([^`]+)`\s*\|") - - -def _section(markdown: str, heading: str) -> str: - level: Final = heading.split(" ", 1)[0] - start: Final = markdown.find(f"\n{heading}\n") - if start < 0: - return "" - body: Final = markdown[start + 1 + len(heading) :] - next_heading: Final = re.search(rf"^{re.escape(level)} ", body, flags=re.MULTILINE) - return body[: next_heading.start()] if next_heading else body - - -def parse_deprecations(markdown: str) -> DeprecationDoc: - redirect_rows: Final = tuple( - m.groups() - for m in (_REDIRECT_ROW.match(line) for line in _section(markdown, "## Active model redirects").splitlines()) - if m - ) - inference: Final = _section(_section(markdown, "## Deprecation history"), "### Inference") - removal_rows: Final = tuple(m.groups() for m in (_REMOVAL_ROW.match(line) for line in inference.splitlines()) if m) - if not redirect_rows or not removal_rows: - raise SyncError( - "deprecations doc parsed to zero redirect or removal rows; the table format at " - f"{DEPRECATIONS_URL} changed and the parser needs updating" - ) - removal_dates: Final = {model: date for date, model in reversed(removal_rows)} - return DeprecationDoc( - removal_dates=MappingProxyType(dict(reversed(removal_dates.items()))), - redirects=MappingProxyType({original: target for original, target in redirect_rows}), - ) - - -def per_token(price_per_million: float) -> float: - return float(f"{price_per_million / 1e6:.6g}") - - -def _resolve_name(name: str, universe: frozenset[str]) -> str | None: - if name in universe: - return name - suffix_matches: Final = tuple(candidate for candidate in universe if candidate.endswith(f"/{name}")) - return suffix_matches[0] if len(suffix_matches) == 1 else None - - -def resolve_successor(model_id: str, doc: DeprecationDoc, live_ids: frozenset[str]) -> str | None: - canonical: Final = live_ids | frozenset(doc.removal_dates) - redirects: Final = { - (_resolve_name(raw_source, canonical) or raw_source): (_resolve_name(raw_target, canonical) or raw_target) - for raw_source, raw_target in doc.redirects.items() - } - seen: Final = set() - current = model_id # rebind-ok: walks the redirect chain - while current in redirects and current not in seen: - seen.add(current) - current = redirects[current] # rebind-ok: walks the redirect chain - return current if current != model_id and current in live_ids else None - - -@dataclass(frozen=True, slots=True) -class SyncOutcome: - cost_map: CostMap - added: tuple[str, ...] = () - updated: tuple[str, ...] = () - deprecated: tuple[str, ...] = () - reappeared: tuple[str, ...] = () - warnings: tuple[str, ...] = () - skipped_types: Mapping[str, int] = field(default_factory=dict) - - @property - def has_changes(self) -> bool: - return bool(self.added or self.updated or self.deprecated or self.reappeared) - - -def _api_fields(model: CatalogModel) -> RegistryEntry: - cached: Final = model.pricing.cached_input - return { - "input_cost_per_token": per_token(model.pricing.input), - "output_cost_per_token": per_token(model.pricing.output), - **({"cache_read_input_token_cost": per_token(cached), "supports_prompt_caching": True} if cached else {}), - **({"max_input_tokens": model.context_length} if model.context_length is not None else {}), - } - - -def _new_entry(model: CatalogModel, mode: str) -> RegistryEntry: - rule: Final = RULES_BY_ID.get(model.id) - legacy_ceiling: Final = {} if model.context_length is None else {"max_tokens": model.context_length} - merged: Final = { - **_api_fields(model), - **legacy_ceiling, - "litellm_provider": PROVIDER, - "mode": mode, - "source": SOURCE_URL, - **(dict(rule.fields) if rule else {}), - } - return dict(sorted(merged.items())) - - -def _updated_entry(entry: RegistryEntry, model: CatalogModel) -> tuple[RegistryEntry, tuple[str, ...]]: - rule: Final = RULES_BY_ID.get(model.id) - desired: Final = {**_api_fields(model), **(dict(rule.fields) if rule else {})} - dropped: Final = () if model.pricing.cached_input else ("cache_read_input_token_cost", "supports_prompt_caching") - changes: Final = tuple( - f"{name}: {entry.get(name)!r} -> {value!r}" for name, value in desired.items() if entry.get(name) != value - ) + tuple( - f"{name}: {entry[name]!r} removed (no longer in the catalog pricing)" for name in dropped if name in entry - ) - merged: Final = {name: value for name, value in {**entry, **desired}.items() if name not in dropped} - return dict(sorted(merged.items())), changes - - -def _with_new_keys_in_block(original: CostMap, result: CostMap, new_keys: Sequence[str]) -> CostMap: - provider_keys: Final = tuple(key for key in original if key.startswith(PREFIX)) - if not new_keys or not provider_keys: - return result - block_end: Final = provider_keys[-1] - return { - key: value - for existing in original - for key, value in ( - (existing, result[existing]), - *((new, result[new]) for new in sorted(new_keys) if existing == block_end), - ) - } - - -def compute_sync(cost_map: CostMap, catalog: Sequence[CatalogModel], doc: DeprecationDoc) -> SyncOutcome: - live_ids: Final = frozenset(model.id for model in catalog) - token_models: Final = {model.id: model for model in catalog if model.type in TYPE_TO_MODE} - skipped: Final = { - model.type: sum(1 for m in catalog if m.type == model.type) - for model in catalog - if model.type not in TYPE_TO_MODE - } - registry_ids: Final = {key.removeprefix(PREFIX): key for key in cost_map if key.startswith(PREFIX)} - - added: Final[list[str]] = [] - updated: Final[list[str]] = [] - deprecated: Final[list[str]] = [] - reappeared: Final[list[str]] = [] - warnings: Final[list[str]] = [] - result: Final[CostMap] = dict(cost_map) - - for model_id, model in sorted(token_models.items()): - mode: Final = TYPE_TO_MODE[model.type] - key: Final = f"{PREFIX}{model_id}" - if model_id in doc.removal_dates: - warnings.append( - f"`{key}` is listed as removed on {doc.removal_dates[model_id]} in the docs but the serverless " - "catalog still serves it; availability kept from the API" - ) - entry = result.get(key) - if not isinstance(entry, dict): - result[key] = _new_entry(model, mode) - added.append(key) - if model.type == "chat" and model_id not in RULES_BY_ID: - warnings.append( - f"`{key}` added without a capability rule; review its tools/vision/reasoning support and add one" - ) - continue - if entry.get("mode") != mode: - warnings.append( - f"`{key}` has curated mode {entry.get('mode')!r} but the catalog maps to {mode!r}; left unchanged" - ) - new_entry, changes = _updated_entry(entry, model) - if "deprecation_date" in new_entry: - new_entry.pop("deprecation_date") - reappeared.append(key) - if changes: - updated.append(f"{key}: " + "; ".join(changes)) - if changes or key in reappeared: - result[key] = new_entry - - for model_id, key in sorted(registry_ids.items()): - if model_id in token_models: - continue - entry = result.get(key) - if not isinstance(entry, dict): - continue - removal_date: Final = doc.removal_dates.get(model_id) - successor: Final = resolve_successor(model_id, doc, live_ids) - metadata = entry.get("metadata") - curated_successor: Final = metadata.get("successor") if isinstance(metadata, dict) else None - new_entry = dict(entry) - if removal_date is not None and entry.get("deprecation_date") != removal_date: - if "deprecation_date" in entry: - warnings.append( - f"`{key}` has curated deprecation_date {entry.get('deprecation_date')!r} but the docs list " - f"{removal_date!r}; left unchanged" - ) - else: - new_entry["deprecation_date"] = removal_date - if removal_date is None and "deprecation_date" not in entry: - warnings.append( - f"`{key}` is absent from the serverless catalog with no removal date in the docs; " - "needs a human deprecation call" - ) - if successor is not None: - desired_successor: Final = f"{PREFIX}{successor}" - if curated_successor is None: - new_entry["metadata"] = dict( - sorted({**(metadata if isinstance(metadata, dict) else {}), "successor": desired_successor}.items()) - ) - elif curated_successor != desired_successor: - warnings.append( - f"`{key}` has curated successor {curated_successor!r} but the docs redirects resolve to " - f"{desired_successor!r}; left unchanged" - ) - if new_entry != entry: - result[key] = dict(sorted(new_entry.items())) - deprecated.append(f"{key}: " + ", ".join(sorted(set(new_entry) - set(entry)) or ["updated"])) - - return SyncOutcome( - cost_map=_with_new_keys_in_block(cost_map, result, tuple(added)), - added=tuple(added), - updated=tuple(updated), - deprecated=tuple(deprecated), - reappeared=tuple(reappeared), - warnings=tuple(warnings), - skipped_types=MappingProxyType(skipped), - ) - - -def _section_block(title: str, lines: Sequence[str], backtick: bool) -> str: - bullets: Final = "\n".join(f"- `{line}`" if backtick else f"- {line}" for line in lines) or "- none" - return f"### {title} ({len(lines)})\n{bullets}\n" - - -def render_pr_body(outcome: SyncOutcome) -> str: - skipped: Final = ", ".join(f"{kind} ({count})" for kind, count in sorted(outcome.skipped_types.items())) or "none" - return ( - "Automated daily sync of the together_ai entries in model_prices_and_context_window.json against " - f"`GET {MODELS_URL}` and {DEPRECATIONS_URL} by scripts/sync_together_ai_models.py.\n" - "\n" - f"{_section_block('Added', outcome.added, backtick=True)}" - "\n" - f"{_section_block('Updated', outcome.updated, backtick=True)}" - "\n" - f"{_section_block('Marked deprecated', outcome.deprecated, backtick=True)}" - "\n" - f"{_section_block('Returned to the catalog', outcome.reappeared, backtick=True)}" - "\n" - f"{_section_block('Warnings needing a human call', outcome.warnings, backtick=False)}" - "\n" - f"Catalog model types outside the sync's token-pricing scope, skipped: {skipped}\n" - ) - - -def render_summary(outcome: SyncOutcome) -> str: - return ( - f"added={len(outcome.added)} updated={len(outcome.updated)} deprecated={len(outcome.deprecated)} " - f"reappeared={len(outcome.reappeared)} warnings={len(outcome.warnings)}" - ) - - -def load_catalog(raw: bytes) -> list[CatalogModel]: - parsed: Final = json.loads(raw) - entries: Final = parsed.get("data") if isinstance(parsed, dict) else parsed - try: - catalog: Final = CATALOG_ADAPTER.validate_python(entries) - except ValidationError as error: - raise SyncError(f"the catalog response no longer matches the expected shape: {error}") from error - if not any(model.type in TYPE_TO_MODE for model in catalog): - raise SyncError( - "the catalog response contains no token-priced models; refusing to mark the whole registry deprecated" - ) - return catalog - - -def _fetch(url: str, headers: Mapping[str, str]) -> bytes: - response: Final = httpx.get(url, headers=dict(headers), timeout=30, follow_redirects=True) - if response.status_code != 200: - raise SyncError(f"GET {url} returned {response.status_code}") - return response.content - - -def _serialize(cost_map: CostMap) -> str: - return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n" - - -def main(argv: Sequence[str]) -> int: - parser: Final = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--write", action="store_true", help="apply the sync to the cost map files (default: dry run)") - parser.add_argument("--models-json", type=Path, help="recorded catalog response to use instead of the live API") - parser.add_argument( - "--deprecations-md", type=Path, help="recorded deprecations doc to use instead of the live docs" - ) - parser.add_argument("--pr-body-file", type=Path, help="write the generated PR body to this path") - parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parent.parent) - args: Final = parser.parse_args(argv) - - if args.models_json is not None: - catalog_raw: Final = args.models_json.read_bytes() - else: - api_key: Final = os.environ.get("TOGETHER_API_KEY") - if not api_key: - raise SyncError("TOGETHER_API_KEY is not set and --models-json was not given") - catalog_raw = _fetch(MODELS_URL, {"Authorization": f"Bearer {api_key}"}) # rebind-ok: branch-dependent source - catalog: Final = load_catalog(catalog_raw) - markdown: Final = ( - args.deprecations_md.read_text() if args.deprecations_md is not None else _fetch(DEPRECATIONS_URL, {}).decode() - ) - doc: Final = parse_deprecations(markdown) - - cost_map_path: Final = args.repo_root / COST_MAP_RELPATHS[0] - cost_map: Final = json.loads(cost_map_path.read_text()) - outcome: Final = compute_sync(cost_map, catalog, doc) - body: Final = render_pr_body(outcome) - - if args.pr_body_file is not None: - args.pr_body_file.write_text(body) - if args.write and outcome.has_changes: - for relpath in COST_MAP_RELPATHS: - (args.repo_root / relpath).write_text(_serialize(outcome.cost_map)) - print(render_summary(outcome)) - print() - print(body) - if not args.write: - print("dry run: no files were touched") - elif not outcome.has_changes: - print("registry already in sync: no files were touched") - return 0 - - -if __name__ == "__main__": - try: - raise SystemExit(main(sys.argv[1:])) - except SyncError as error: - print(f"SYNC FAILED: {error}", file=sys.stderr) - raise SystemExit(1) from error diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index b5e36055354..853b10ad97b 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -171,6 +171,7 @@ ignored_function_names = [ "_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py "_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py "_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py + "_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py ] diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index e3994e598c6..ea095d4510c 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -71,9 +71,9 @@ - {id: llm.messages.anthropic.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching via Messages API"} - {id: llm.messages.anthropic.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Extended thinking via Messages API"} - {id: llm.messages.bedrock_converse.basic.nonstream.works_for_kimi_k3_max_tokens_1, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works_for_kimi_k3_max_tokens_1], source: "llm_translation/test_bedrock_kimi_k3_messages_e2e.py", rationale: "Bedrock Converse accepts max_tokens=1 on the Anthropic Messages surface after LiteLLM applies the model minimum", fail_before_fix: proven} -- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven} -- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (#32831)", fail_before_fix: proven} -- {id: llm.messages.bedrock_invoke.web_search_server_tool.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: web_search_server_tool, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Bedrock hosts no web_search server tool, so this only works because interception rewrites it before the upstream call and the agentic loop feeds the results back in native shape; a regression that short-circuits or forwards it instead yields raw text or AWS's 400", fail_before_fix: unproven} +- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven} +- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (#32831)", fail_before_fix: proven} +- {id: llm.messages.bedrock_invoke.web_search_server_tool.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: web_search_server_tool, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py", rationale: "Bedrock hosts no web_search server tool, so this only works because interception rewrites it before the upstream call and the agentic loop feeds the results back in native shape; a regression that short-circuits or forwards it instead yields raw text or AWS's 400", fail_before_fix: unproven} - {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven} - {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven} - {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven} diff --git a/tests/e2e/ui/globalSetup.ts b/tests/e2e/ui/globalSetup.ts index d97789c227e..e18d1be390c 100644 --- a/tests/e2e/ui/globalSetup.ts +++ b/tests/e2e/ui/globalSetup.ts @@ -2,7 +2,7 @@ import { chromium, expect, request } from "@playwright/test"; import { users, Role, STORAGE_PATHS } from "./fixtures/users"; import { ARTIFACT_DIR, UI_BASE_URL } from "./constants"; import { expectUnrestrictedDashboard, setInvitedUserPassword } from "./helpers/userOnboarding"; -import { hideLiteAdmin } from "./helpers/navigation"; +import { setAriaHidden } from "./helpers/navigation"; import { masterKey as getMasterKey } from "./helpers/traffic"; import * as fs from "fs"; import * as path from "path"; @@ -78,7 +78,7 @@ async function globalSetup() { await dismiss.click(); } if (role === Role.ProxyAdmin) { - await hideLiteAdmin(page); + await setAriaHidden(page, true); } // The login flow stores a post-login return URL in the litellm_return_url // cookie. If the snapshot captures it before the app consumes it, every diff --git a/tests/e2e/ui/helpers/navigation.ts b/tests/e2e/ui/helpers/navigation.ts index 78288fab743..be46fcf0395 100644 --- a/tests/e2e/ui/helpers/navigation.ts +++ b/tests/e2e/ui/helpers/navigation.ts @@ -62,18 +62,29 @@ export async function dismissFeedbackPopup(page: PlaywrightPage): Promise } } -export async function hideLiteAdmin(page: PlaywrightPage): Promise { +export async function setAriaHidden(page: PlaywrightPage, hidden: boolean): Promise { await page.getByRole("button", { name: /Account menu/i }).click(); const panel = page.getByTestId("sidebar-account-menu-panel"); await expect(panel).toBeVisible({ timeout: 5_000 }); - const toggle = panel.getByRole("switch", { name: "Toggle hide LiteAdmin" }); - if ((await toggle.getAttribute("aria-checked")) !== "true") { - await toggle.click(); + const premium = await panel.getByText("Premium", { exact: true }).isVisible(); + const toggle = panel.getByRole("switch", { name: "Toggle hide Aria Agent", exact: true }); + if (premium) { + await expect(toggle).toBeVisible(); + if ((await toggle.getAttribute("aria-checked")) !== String(hidden)) { + await toggle.click(); + } + await expect(toggle).toHaveAttribute("aria-checked", String(hidden)); + } else { + await expect(toggle).toHaveCount(0); } - await expect(toggle).toHaveAttribute("aria-checked", "true"); await page.keyboard.press("Escape"); await expect(panel).toBeHidden(); - await expect(page.getByRole("button", { name: "LiteAdmin", exact: true })).toBeHidden(); + const aria = page.getByRole("banner").getByRole("button", { name: "Aria Agent", exact: true }); + if (premium && hidden) { + await expect(aria).toBeHidden(); + } else { + await expect(aria).toBeVisible(); + } } /** diff --git a/tests/e2e/ui/tests/logs/logsPagination.spec.ts b/tests/e2e/ui/tests/logs/logsPagination.spec.ts index 416e1c3171f..838a829a3e2 100644 --- a/tests/e2e/ui/tests/logs/logsPagination.spec.ts +++ b/tests/e2e/ui/tests/logs/logsPagination.spec.ts @@ -1,8 +1,14 @@ import { test, expect, type APIRequestContext, type Locator, type Page as PlaywrightPage } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; -import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; +import { navigateToPage, dismissFeedbackPopup, setAriaHidden } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; -import { CHAT_MODEL_A, createVirtualKey, sendChatCompletion, waitForSpendLog } from "../../helpers/traffic"; +import { + CHAT_MODEL_A, + createVirtualKey, + deleteVirtualKey, + sendChatCompletion, + waitForSpendLog, +} from "../../helpers/traffic"; /** * Session-grouped pagination (#38060): a page of N rows must render exactly N session rows, a @@ -45,6 +51,57 @@ async function setRowsPerPage(page: PlaywrightPage, size: "25" | "50" | "100"): await page.getByRole("option", { name: size, exact: true }).click(); } +test.describe("Logs pagination with Aria in the header", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + for (const viewport of [ + { width: 1440, height: 900 }, + { width: 390, height: 844 }, + ]) { + test(`keeps pagination reachable and Aria available at ${viewport.width}px`, async ({ page, request }) => { + await page.setViewportSize(viewport); + const alias = `e2e-aria-header-${uniqueSuffix()}`; + const key = await createVirtualKey(request, { key_alias: alias }); + try { + const ids = await Promise.all( + Array.from({ length: 26 }, (_, index) => + sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt: `${alias}-${index}`, + apiKey: key.key, + }), + ), + ); + await Promise.all(ids.map((id) => waitForSpendLog(request, id))); + await openLogs(page); + await setAriaHidden(page, false); + await applyKeyAliasFilter(page, await openFilterDrawer(page), alias); + await setRowsPerPage(page, "25"); + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26"); + await visibleTestId(page, "pagination-next").click(); + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 26-26 of 26"); + + const aria = page.getByRole("banner").getByRole("button", { name: "Aria Agent", exact: true }); + await aria.click(); + await expect(page.getByRole("dialog", { name: "Aria Agent", exact: true })).toBeVisible(); + await page.keyboard.press("Escape"); + await expect(aria).toBeFocused(); + await visibleTestId(page, "pagination-first").click(); + await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26"); + for (const destination of [Page.Usage, Page.LlmPlayground, Page.ApiKeys]) { + await navigateToPage(page, destination); + await aria.click(); + await expect(page.getByRole("dialog", { name: "Aria Agent", exact: true })).toBeVisible(); + await page.keyboard.press("Escape"); + await expect(aria).toBeFocused(); + } + } finally { + await deleteVirtualKey(request, key.token); + } + }); + } +}); + test.describe("Logs page session-grouped pagination", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); diff --git a/tests/integration/_support/copilot_session_prometheus_worker.py b/tests/integration/_support/copilot_session_prometheus_worker.py new file mode 100644 index 00000000000..a6b2c5d4692 --- /dev/null +++ b/tests/integration/_support/copilot_session_prometheus_worker.py @@ -0,0 +1,162 @@ +"""One SDK process for the SDK provider-401 metric cells: a provider 401 on a caller's own Copilot session, one on +the router deployment's own key, one on a key the router caller passes and one on a direct call's own key, then a +report of the deployment series on that process's default registry. Run as a script by ``run_copilot_session_worker`` so each process registers the +Prometheus metrics exactly once.""" + +from __future__ import annotations + +import asyncio +import json +import subprocess +import sys +from itertools import chain +from dataclasses import dataclass +from typing import Final + +import litellm +import openai +from litellm.integrations.prometheus import PrometheusLogger +from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession +from prometheus_client import REGISTRY +from pydantic import TypeAdapter + +COPILOT_DEPLOYMENT: Final = "copilot-per-user-deployment" +OPERATOR_DEPLOYMENT: Final = "operator-key-deployment" +CALLER_SESSION_TOKEN: Final = "caller-copilot-session-token" +OPERATOR_KEY: Final = "rotated-operator-key" +ROUTER_CALLER_KEY: Final = "rotated-router-caller-key" +DIRECT_CALL: Final = "direct-sdk-call" +DIRECT_KEY: Final = "rotated-direct-call-key" +FAILURES: Final = "litellm_deployment_failure_responses_total" +TOTALS: Final = "litellm_deployment_total_requests_total" +LLM_FAILED: Final = "litellm_llm_api_failed_requests_metric_total" +STATE: Final = "litellm_deployment_state" +DEPLOYMENT_SERIES: Final = frozenset({FAILURES, TOTALS, LLM_FAILED, STATE}) +WORKER_TIMEOUT_SECONDS: Final = 120 + + +@dataclass(frozen=True, slots=True) +class Outcome: + deployment: str + status: int | None + + +@dataclass(frozen=True, slots=True) +class Series: + metric: str + model_id: str + api_base: str + value: float + + +@dataclass(frozen=True, slots=True) +class WorkerReport: + api_base: str + outcomes: tuple[Outcome, ...] + series: tuple[Series, ...] + + def values(self, metric: str, model_id: str) -> tuple[float, ...]: + return tuple(s.value for s in self.series if s.metric == metric and s.model_id == model_id) + + def on_the_endpoint(self, metric: str, model_id: str) -> tuple[float, ...]: + return tuple( + s.value for s in self.series if (s.metric, s.model_id, s.api_base) == (metric, model_id, self.api_base) + ) + + def outside_the_router(self, metric: str) -> tuple[float, ...]: + return tuple(s.value for s in self.series if s.metric == metric and not s.model_id) + + def on_a_one_off_deployment(self, metric: str) -> tuple[float, ...]: + named_ids: Final = ("", COPILOT_DEPLOYMENT, OPERATOR_DEPLOYMENT) + return tuple(s.value for s in self.series if s.metric == metric and s.model_id not in named_ids) + + +_REPORT: Final = TypeAdapter(WorkerReport) + + +def run_copilot_session_worker(api_base: str) -> WorkerReport: + completed: Final = subprocess.run( + (sys.executable, "-P", __file__, api_base), + capture_output=True, + text=True, + timeout=WORKER_TIMEOUT_SECONDS, + check=False, + ) + assert completed.returncode == 0, completed.stderr[-3000:] + return _REPORT.validate_json(completed.stdout.splitlines()[-1]) + + +def _deployment_series() -> tuple[Series, ...]: + samples: Final = tuple(chain.from_iterable(family.samples for family in REGISTRY.collect())) + return tuple( + Series( + metric=sample.name, + model_id=str(sample.labels.get("model_id", "")), + api_base=str(sample.labels.get("api_base", "")), + value=float(sample.value), + ) + for sample in samples + if sample.name in DEPLOYMENT_SERIES + ) + + +def _router(api_base: str) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": COPILOT_DEPLOYMENT, + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": "copilot-per-user", + "github_copilot_auth_type": "per_user_oauth", + }, + "model_info": {"id": COPILOT_DEPLOYMENT}, + }, + { + "model_name": OPERATOR_DEPLOYMENT, + "litellm_params": {"model": "openai/gpt-5.6", "api_base": api_base, "api_key": OPERATOR_KEY}, + "model_info": {"id": OPERATOR_DEPLOYMENT}, + }, + ], + num_retries=0, + disable_cooldowns=True, + ) + + +async def _attempt(router: litellm.Router, model: str, **extra: object) -> Outcome: + try: + await router.acompletion(model=model, messages=[{"role": "user", "content": f"{model} worker"}], **extra) + except openai.APIStatusError as error: + return Outcome(deployment=model, status=error.status_code) + return Outcome(deployment=model, status=None) + + +async def _direct_attempt(api_base: str) -> Outcome: + try: + await litellm.acompletion( + model="openai/gpt-5.6", + api_base=api_base, + api_key=DIRECT_KEY, + messages=[{"role": "user", "content": f"{DIRECT_CALL} worker"}], + ) + except openai.APIStatusError as error: + return Outcome(deployment=DIRECT_CALL, status=error.status_code) + return Outcome(deployment=DIRECT_CALL, status=None) + + +async def main(api_base: str) -> None: + litellm.callbacks = [PrometheusLogger()] # test-quality-ok: this worker process exits right after the report + session: Final = GithubCopilotUserSession(token=CALLER_SESSION_TOKEN, api_base=api_base) + router: Final = _router(api_base) + outcomes: Final = ( + await _attempt(router, COPILOT_DEPLOYMENT, github_copilot_user_session=session), + await _attempt(router, OPERATOR_DEPLOYMENT), + await _attempt(router, OPERATOR_DEPLOYMENT, api_key=ROUTER_CALLER_KEY), + await _direct_attempt(api_base), + ) + report: Final = WorkerReport(api_base=api_base, outcomes=outcomes, series=_deployment_series()) + sys.stdout.write(json.dumps(_REPORT.dump_python(report)) + "\n") + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1])) diff --git a/tests/integration/authorization/test_key_budget_owner_updates.py b/tests/integration/authorization/test_key_budget_owner_updates.py new file mode 100644 index 00000000000..8415c044b6f --- /dev/null +++ b/tests/integration/authorization/test_key_budget_owner_updates.py @@ -0,0 +1,195 @@ +from collections.abc import Iterator +from hashlib import sha256 +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest +import yaml +from integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + eventually, + gateway_from_environment, + string_value, +) +from integration._support.process import owned_proxy +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.shares_worker_state + +CAP: Final = 0.1 +BUDGET_EXCEEDED: Final = 422 +FAST_RESET_TICK: Final = MappingProxyType( + {"PROXY_BUDGET_RESCHEDULER_MIN_TIME": "2", "PROXY_BUDGET_RESCHEDULER_MAX_TIME": "2"} +) + + +@pytest.fixture(scope="module") +def resetting(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + with ( + gateway_from_environment() as shared, + owned_proxy(shared, tmp_path_factory.mktemp("fast-reset"), FAST_RESET_TICK) as replica, + ): + yield replica + + +def _policy_config(directory: Path, policy: str) -> Path: + shared: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + general_settings: Final = { + **JSON_OBJECT.validate_python(shared["general_settings"]), + "self_serve_budget_policy": policy, + } + path: Final = directory / f"{policy}.yaml" + path.write_text(yaml.safe_dump({**shared, "general_settings": general_settings})) + return path + + +@pytest.fixture(scope="module", params=("lower_only", "ceiling")) +def self_serve( + request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[tuple[str, Gateway]]: + policy: Final = string_value(request.param) + directory: Final = tmp_path_factory.mktemp(policy) + with ( + gateway_from_environment() as shared, + owned_proxy(shared, directory, {}, config=_policy_config(directory, policy)) as replica, + ): + yield policy, replica + + +def _owner_credential(scenario: Scenario) -> str: + return scenario.key(user_id=scenario.user(user_role="internal_user")) + + +def _owned_key(gateway: Gateway, scenario: Scenario, owner: str, **fields: JsonValue) -> str: + response: Final = gateway.request("POST", "/key/generate", fields, key=owner) + assert response.status_code == 200, response.text + token: Final = string_value(JSON_OBJECT.validate_json(response.content)["key"]) + scenario.cleanups.callback(scenario.delete_key, token) + return token + + +def _key_row(gateway: Gateway, key: str) -> dict[str, JsonValue]: + rows: Final = gateway.rows( + "SELECT spend, max_budget, budget_duration, budget_id, budget_reset_at::text AS reset_at" + ' FROM "LiteLLM_VerificationToken" WHERE token = %s', + (sha256(key.encode()).hexdigest(),), + ) + assert len(rows) == 1, rows + return rows[0] + + +def _exhaust(gateway: Gateway, key: str) -> None: + gateway.post("/key/update", {"key": key, "spend": CAP}) + assert _key_row(gateway, key)["spend"] == CAP + + +def _chat_status(gateway: Gateway, model: str, key: str) -> int: + return gateway.request( + "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": "budget"}]}, key=key + ).status_code + + +def test_key_owner_cannot_shorten_budget_duration_to_restart_an_exhausted_budget(resetting: Gateway) -> None: + with resetting.scenario() as scenario: + model: Final = scenario.model() + owner: Final = _owner_credential(scenario) + target: Final = _owned_key(resetting, scenario, owner, models=[model], max_budget=CAP, budget_duration="30d") + control: Final = _owned_key(resetting, scenario, owner, models=[model], max_budget=CAP, budget_duration="30d") + _exhaust(resetting, target) + _exhaust(resetting, control) + assert _chat_status(resetting, model, target) == BUDGET_EXCEEDED + before: Final = _key_row(resetting, target) + + denied: Final = resetting.request("POST", "/key/update", {"key": target, "budget_duration": "1s"}, key=owner) + + resetting.post("/key/update", {"key": control, "budget_duration": "1s"}) + eventually(lambda: _key_row(resetting, control)["spend"], lambda spend: spend == 0.0, seconds=60) + assert denied.status_code == 403, denied.text + assert _key_row(resetting, target) == before + assert _chat_status(resetting, model, target) == BUDGET_EXCEEDED + + +def test_key_owner_cannot_swap_or_reschedule_a_tier_key(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + low: Final = scenario.budget(max_budget=CAP) + high: Final = scenario.budget(max_budget=100.0) + owner: Final = _owner_credential(scenario) + target: Final = _owned_key(gateway, scenario, owner, models=[model], budget_id=low) + _exhaust(gateway, target) + assert _chat_status(gateway, model, target) == BUDGET_EXCEEDED + before: Final = _key_row(gateway, target) + + swapped: Final = gateway.request("POST", "/key/update", {"key": target, "budget_id": high}, key=owner) + rescheduled: Final = gateway.request("POST", "/key/update", {"key": target, "budget_duration": "1s"}, key=owner) + + assert swapped.status_code == 403, swapped.text + assert rescheduled.status_code == 403, rescheduled.text + assert _key_row(gateway, target) == before + assert _chat_status(gateway, model, target) == BUDGET_EXCEEDED + + +def test_team_member_cannot_shorten_budget_duration_on_a_team_key(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team(team_member_permissions=["/key/update", "/key/info"]) + member: Final = scenario.key(user_id=scenario.member(team), team_id=team) + target: Final = scenario.key(team_id=team, max_budget=CAP, budget_duration="30d") + before: Final = _key_row(gateway, target) + + denied: Final = gateway.request("POST", "/key/update", {"key": target, "budget_duration": "1s"}, key=member) + + assert denied.status_code == 403, denied.text + assert _key_row(gateway, target) == before + + +def test_key_owner_keeps_edits_that_cannot_restart_a_budget(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + owner: Final = _owner_credential(scenario) + capped: Final = _owned_key(gateway, scenario, owner, max_budget=CAP, budget_duration="monthly") + uncapped: Final = _owned_key(gateway, scenario, owner) + + renamed: Final = gateway.request("POST", "/key/update", {"key": capped, "key_alias": "renamed"}, key=owner) + resent: Final = gateway.request("POST", "/key/update", {"key": capped, "budget_duration": "30d"}, key=owner) + scheduled: Final = gateway.request("POST", "/key/update", {"key": uncapped, "budget_duration": "1d"}, key=owner) + + assert renamed.status_code == 200, renamed.text + assert resent.status_code == 200, resent.text + assert scheduled.status_code == 200, scheduled.text + assert _key_row(gateway, uncapped)["budget_duration"] == "1d" + + +def test_self_serve_policy_keeps_tier_keys_admin_only(self_serve: tuple[str, Gateway]) -> None: + _, proxy = self_serve + with proxy.scenario() as scenario: + low: Final = scenario.budget(max_budget=CAP) + high: Final = scenario.budget(max_budget=100.0) + owner: Final = _owner_credential(scenario) + target: Final = _owned_key(proxy, scenario, owner, budget_id=low) + before: Final = _key_row(proxy, target) + + swapped: Final = proxy.request("POST", "/key/update", {"key": target, "budget_id": high}, key=owner) + rescheduled: Final = proxy.request("POST", "/key/update", {"key": target, "budget_duration": "1d"}, key=owner) + + assert swapped.status_code == 403, swapped.text + assert rescheduled.status_code == 403, rescheduled.text + assert _key_row(proxy, target) == before + + +def test_self_serve_policy_gates_a_new_reset_period_like_a_raised_cap(self_serve: tuple[str, Gateway]) -> None: + policy, proxy = self_serve + with proxy.scenario() as scenario: + owner: Final = _owner_credential(scenario) + shortened: Final = _owned_key(proxy, scenario, owner, max_budget=CAP, budget_duration="30d") + cleared: Final = _owned_key(proxy, scenario, owner, max_budget=CAP, budget_duration="30d") + + shorten: Final = proxy.request("POST", "/key/update", {"key": shortened, "budget_duration": "1s"}, key=owner) + clear: Final = proxy.request("POST", "/key/update", {"key": cleared, "budget_duration": None}, key=owner) + + assert (shorten.status_code, _key_row(proxy, shortened)["budget_duration"]) == ( + (403, "30d") if policy == "lower_only" else (200, "1s") + ), shorten.text + assert clear.status_code == 200, clear.text + assert _key_row(proxy, cleared)["budget_duration"] is None diff --git a/tests/integration/mcp/test_mcp_oauth_flows.py b/tests/integration/mcp/test_mcp_oauth_flows.py index 327b053d333..90a462472b2 100644 --- a/tests/integration/mcp/test_mcp_oauth_flows.py +++ b/tests/integration/mcp/test_mcp_oauth_flows.py @@ -792,17 +792,23 @@ def _ui_session_cookie(gateway: Gateway, user_id: str) -> dict[str, str]: def _gateway_session_sign_in( - gateway: Gateway, user_id: str, resource: str | None = None + gateway: Gateway, + user_id: str, + resource: str | None = None, + *, + registration_endpoint: str = "/register", + authorization_endpoint: str = "/authorize", + token_endpoint: str = "/token", ) -> tuple[dict[str, object], str]: registered: Final = gateway.client.post( - "/register", json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} + registration_endpoint, json={"redirect_uris": [CLIENT_REDIRECT], "client_name": "integration"} ) assert registered.status_code in (200, 201), registered.text client_id: Final = str(_response_object(registered)["client_id"]) pkce: Final = _Pkce(secrets.token_urlsafe(48)) cookies: Final = _ui_session_cookie(gateway, user_id) started: Final = gateway.client.get( - "/authorize", + authorization_endpoint, params={ "client_id": client_id, "redirect_uri": CLIENT_REDIRECT, @@ -823,13 +829,14 @@ def _gateway_session_sign_in( callback: Final = parse_qs(urlsplit(completed.headers["location"]).query) assert "code" in callback, completed.headers["location"] issued: Final = gateway.client.post( - "/token", + token_endpoint, data={ "grant_type": "authorization_code", "code": callback["code"][0], "redirect_uri": CLIENT_REDIRECT, "client_id": client_id, "code_verifier": pkce.verifier, + **({} if resource is None else {"resource": resource}), }, ) assert issued.status_code == 200, issued.text @@ -1292,10 +1299,18 @@ def test_lite_logout_revokes_the_cli_refresh_token_sent_the_way_the_cli_sends_it assert completion.status_code == 200, completion.text -def _toolset_rpc(gateway: Gateway, bearer: str, name: str, method: str, params: dict[str, object]) -> Outcome: +def _toolset_rpc( + gateway: Gateway, + bearer: str, + name: str, + method: str, + params: dict[str, object], + *, + route: str = "/toolset/{name}/mcp", +) -> Outcome: def post(rpc_method: str, rpc_params: dict[str, object]) -> httpx.Response: return gateway.client.post( - f"/toolset/{name}/mcp", + route.format(name=name), json={"jsonrpc": "2.0", "id": 1, "method": rpc_method, "params": rpc_params}, headers={"Authorization": f"Bearer {bearer}", "Accept": "application/json, text/event-stream"}, ) @@ -1306,7 +1321,12 @@ def _toolset_rpc(gateway: Gateway, bearer: str, name: str, method: str, params: return _outcome_from_rpc(post(method, params)) -def test_gateway_session_bearer_of_a_team_member_is_served_the_team_toolset_on_its_route(gateway: Gateway) -> None: +@pytest.mark.parametrize("route", ["/toolset/{name}/mcp", "/{name}/mcp"]) +def test_gateway_session_bearer_of_a_team_member_is_served_the_team_toolset_on_its_route( + gateway: Gateway, route: str +) -> None: + from mcp.shared.auth_utils import check_resource_allowed + with mcp_peer() as peer, gateway.scenario() as scenario: alias: Final = "lit6029sess" + uuid.uuid4().hex[:6] server_id: Final = register_mcp(scenario, peer, alias) @@ -1315,19 +1335,69 @@ def test_gateway_session_bearer_of_a_team_member_is_served_the_team_toolset_on_i granted_id: Final = create_toolset(scenario, ((server_id, "add"),), toolset_name=granted_name) create_toolset(scenario, ((server_id, "multiply"),), toolset_name=withheld_name) member: Final = scenario.member(scenario.team(object_permission={"mcp_toolsets": [granted_id]})) - bearer: Final = _gateway_session_bearer(gateway, member) + path: Final = route.format(name=granted_name) + anonymous: Final = gateway.client.post( + path, + headers=ACCEPT, + json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": dict(INITIALIZE)}, + ) + assert anonymous.status_code == 401, anonymous.text + metadata_url: Final = anonymous.headers["www-authenticate"].split('resource_metadata="', 1)[1].split('"', 1)[0] + protected: Final = gateway.client.get(metadata_url) + assert protected.status_code == 200, protected.text + protected_body: Final = _response_object(protected) + resource: Final = str(protected_body["resource"]) + assert check_resource_allowed( + requested_resource=f"{_base(gateway)}{path}", configured_resource=resource + ), protected.text + issuers: Final = TypeAdapter(tuple[str, ...]).validate_python(protected_body["authorization_servers"]) + assert len(issuers) == 1, protected.text + issuer: Final = urlsplit(issuers[0]) + authorization: Final = gateway.client.get( + f"{issuer.scheme}://{issuer.netloc}/.well-known/oauth-authorization-server{issuer.path}" + ) + assert authorization.status_code == 200, authorization.text + authorization_body: Final = _response_object(authorization) + assert authorization_body["issuer"] == issuers[0], authorization.text + token_endpoint: Final = str(authorization_body["token_endpoint"]) + issued, client_id = _gateway_session_sign_in( + gateway, + member, + resource, + registration_endpoint=str(authorization_body["registration_endpoint"]), + authorization_endpoint=str(authorization_body["authorization_endpoint"]), + token_endpoint=token_endpoint, + ) + bearer: Final = _issued_token(issued) assert bearer.startswith("llm_session_"), bearer[:16] - listed: Final = _toolset_rpc(gateway, bearer, granted_name, "tools/list", {}) + listed: Final = _toolset_rpc(gateway, bearer, granted_name, "tools/list", {}, route=route) assert listed.tools == (f"{alias}-add",), listed.raw peer.drain() called: Final = _toolset_rpc( - gateway, bearer, granted_name, "tools/call", {"name": f"{alias}-add", "arguments": {"a": 4, "b": 5}} + gateway, bearer, granted_name, "tools/call", + {"name": f"{alias}-add", "arguments": {"a": 4, "b": 5}}, route=route ) assert called.ok and called.text == "9", called.raw assert len(tool_calls(peer.drain())) == 1 - denied: Final = _toolset_rpc(gateway, bearer, withheld_name, "tools/list", {}) + denied: Final = _toolset_rpc(gateway, bearer, withheld_name, "tools/list", {}, route=route) assert denied.status == 403, denied.raw assert tool_calls(peer.drain()) == () + rotated: Final = gateway.client.post( + token_endpoint, + data={ + "grant_type": "refresh_token", + "refresh_token": str(issued["refresh_token"]), + "client_id": client_id, + "resource": resource, + }, + ) + assert rotated.status_code == 200, rotated.text + rotated_body: Final = _response_object(rotated) + assert rotated_body["refresh_token"] != issued["refresh_token"], rotated.text + refreshed: Final = _toolset_rpc( + gateway, _issued_token(rotated_body), granted_name, "tools/list", {}, route=route + ) + assert refreshed.tools == listed.tools, refreshed.raw def test_resource_scoped_session_bearer_opens_a_team_toolset_inside_its_server_and_none_outside( diff --git a/tests/integration/observability/test_otel_team_sampling_across_pods.py b/tests/integration/observability/test_otel_team_sampling_across_pods.py new file mode 100644 index 00000000000..dc7b295febd --- /dev/null +++ b/tests/integration/observability/test_otel_team_sampling_across_pods.py @@ -0,0 +1,723 @@ +"""A team's Arize sampling rate decides once per trace, so every pod and worker serving a call of it agrees. + +Two proxies share one database, as two pods behind a load balancer do: the first runs one worker, the second +two. Calls of one trace carry the caller's ``traceparent`` and land on different pods and workers; the team's +Arize space must receive every call of a trace or none of them. +""" + +import asyncio +import json +import re +import uuid +from collections.abc import Callable, Generator, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass, replace +from itertools import chain, product +from pathlib import Path +from typing import Final, Literal, TypeAlias +from urllib.parse import urlparse + +import httpx +import pytest +import yaml +from integration._support.client import ( + Scenario, + eventually, + gateway_from_environment, + list_value, + object_value, + until_asserted, +) +from integration._support.database import scratch_database +from integration._support.otlp_sink import Span, SpanSinks, configure_sink, recorded_requests, recorded_spans +from integration._support.process import OwnedProxy, WorkerHealthcheck, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +MODEL: Final = "team-sampling" +SERVER: Final = 2 +CALL_ID: Final = "litellm.call_id" +OPERATION: Final = "gen_ai.operation.name" +SPACE: Final = "space-sampled" +CREDENTIALS: Final = {"arize_space_key": SPACE, "arize_api_key": "arize-sampled"} +FAILING: Final = re.compile(rb"sampling-fail") +JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +TRACES: Final = 16 +Endpoint: TypeAlias = Literal["chat", "responses", "messages"] +Verdicts: TypeAlias = Mapping[str, tuple[bool, ...]] + + +def _chat_reply(stream: bool) -> Reply: + usage: Final[JsonValue] = {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9} + if not stream: + return Reply( + body=json.dumps( + { + "id": "chatcmpl-sampling", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], + "usage": usage, + } + ).encode() + ) + chunk: Final = {"id": "chatcmpl-sampling", "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"} + deltas: Final[tuple[JsonValue, ...]] = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]}, + {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**chunk, "choices": [], "usage": usage}, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(b"data: " + json.dumps(delta).encode() + b"\n\n" for delta in deltas), b"data: [DONE]\n\n"), + ) + + +def _responses_reply(stream: bool) -> Reply: + response: Final[Mapping[str, JsonValue]] = { + "id": "resp_sampling", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": "msg_sampling", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 7, "output_tokens": 2, "total_tokens": 9}, + } + if not stream: + return Reply(body=json.dumps(response).encode()) + events: Final[tuple[Mapping[str, JsonValue], ...]] = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**response, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_sampling", + "output_index": 0, + "content_index": 0, + "delta": "ok", + }, + {"type": "response.completed", "sequence_number": 2, "response": dict(response)}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events), + ) + + +def _upstream(request: Request) -> Reply: + if FAILING.search(request.body) is not None: + return Reply(status=500, body=b'{"error":{"message":"scripted upstream failure","type":"server_error"}}') + stream: Final = object_value(JSON.validate_json(request.body)).get("stream") is True + if request.target.endswith("/responses"): + return _responses_reply(stream) + return _chat_reply(stream) + + +@dataclass(frozen=True, slots=True) +class Sent: + trace_id: str + parent_span_id: str + call_id: str + + +@dataclass(frozen=True, slots=True) +class Call: + path: str + body: Mapping[str, JsonValue] + headers: Mapping[str, str] + sent: Sent + expected: int + + +def _call( + key: str, trace_id: str | None, endpoint: Endpoint = "chat", stream: bool = False, content: str = "hello" +) -> Call: + parent: Final = uuid.uuid4().hex[:16] + call_id: Final = str(uuid.uuid4()) + messages: Final[JsonValue] = [{"role": "user", "content": content}] + path, body = { + "chat": ("/v1/chat/completions", {"model": MODEL, "messages": messages, "stream": stream}), + "responses": ("/v1/responses", {"model": MODEL, "input": content, "stream": stream}), + "messages": ("/v1/messages", {"model": MODEL, "max_tokens": 16, "messages": messages, "stream": stream}), + }[endpoint] + headers: Final = { + "Authorization": f"Bearer {key}", + "x-litellm-call-id": call_id, + "connection": "close", + **({} if trace_id is None else {"traceparent": f"00-{trace_id}-{parent}-01"}), + } + expected: Final = 500 if FAILING.search(content.encode()) else 200 + return Call(path, body, headers, Sent(trace_id or "", parent if trace_id else "", call_id), expected) + + +@dataclass(frozen=True, slots=True) +class Pods: + """``first`` serves with one worker, ``second`` with two.""" + + first: OwnedProxy + second: OwnedProxy + scenario: Scenario + sinks: SpanSinks + + def team_key(self, variables: Mapping[str, str]) -> str: + team: Final = self.scenario.team() + attached: Final = self.first.gateway.request( + "POST", f"/team/{team}/callback", {"callback_name": "arize", "callback_vars": dict(variables)} + ) + assert attached.status_code == 200, attached.text + return self.scenario.key(team_id=team) + + def send(self, pod: OwnedProxy, call: Call) -> Sent: + with pod.gateway.client.stream("POST", call.path, json=call.body, headers=call.headers) as response: + text: Final = response.read().decode() + assert response.status_code == call.expected, text + return call.sent + + async def send_async(self, client: httpx.AsyncClient, call: Call) -> Sent: + async with client.stream("POST", call.path, json=call.body, headers=call.headers) as response: + text: Final = (await response.aread()).decode() + assert response.status_code == call.expected, text + return call.sent + + def on_each_worker(self, pod: OwnedProxy, call: Callable[[], Call]) -> tuple[Sent, ...]: + """One call served by each worker of ``pod``, the others suspended while it runs.""" + workers: Final = sorted(pod.worker_pids()) + assert len(workers) == pod.gateway.workers, workers + sent: Final[list[Sent]] = [] # mutable-ok: one call per worker, in worker order + for serving in workers: + with _suspended_all_but(pod, workers, serving): + sent.append(self.send(pod, call())) + return tuple(sent) + + def fence(self, key: str) -> tuple[Sent, ...]: + """Calls of an unsampled team on the sampled team's exporter, one per worker of both pods. + + Each worker sends spans to that exporter in the order they were handed to it, so all of these + arriving means every call any worker decided before them has been sent too. + """ + return (self.send(self.first, _call(key, uuid.uuid4().hex)),) + self.on_each_worker( + self.second, lambda: _call(key, uuid.uuid4().hex) + ) + + +@contextmanager +def _suspended_all_but(pod: OwnedProxy, workers: Sequence[int], serving: int) -> Generator[None]: + if len(workers) == 1: + yield + return + others: Final = [pid for pid in workers if pid != serving] + with pod.suspended(others[0]): + yield + + +def _space_id(request: Mapping[str, JsonValue]) -> JsonValue: + return {name.lower(): value for name, value in object_value(request.get("headers") or {}).items()}.get( + "arize-space-id" + ) + + +def _space_span_ids(url: str) -> frozenset[str]: + """Span ids the Arize sink received in export requests sent with the team's space id.""" + ours: Final = ( + list_value(request.get("span_ids") or []) for request in recorded_requests(url) if _space_id(request) == SPACE + ) + return frozenset(span_id for span_id in chain.from_iterable(ours) if isinstance(span_id, str)) + + +def _team_spans(pods: Pods, since: int) -> tuple[Span, ...]: + owned: Final = _space_span_ids(pods.sinks.arize) + return tuple(span for span in recorded_spans(pods.sinks.arize, since)[1] if span["span_id"] in owned) + + +def _delivered(spans: tuple[Span, ...], sent: Sequence[Sent]) -> frozenset[str]: + """Call ids of ``sent`` whose server span is among ``spans``.""" + by_parent: Final = {(call.trace_id, call.parent_span_id): call.call_id for call in sent} + return frozenset( + by_parent[(span["trace_id"], span["parent_span_id"])] + for span in spans + if span["kind"] == SERVER and (span["trace_id"], span["parent_span_id"]) in by_parent + ) + + +def _await_logged(pods: Pods, since: int, sent: Sequence[Sent]) -> None: + """Wait until every call of ``sent`` has its model span in the operator's unsampled copy.""" + expected: Final = {call.call_id for call in sent} + eventually( + lambda: ( + { + str(span["attributes"][CALL_ID]) + for span in recorded_spans(pods.sinks.operator, since)[1] + if OPERATION in span["attributes"] and CALL_ID in span["attributes"] + } + & expected + ), + lambda logged: logged == expected, + seconds=60, + ) + + +@dataclass(frozen=True, slots=True) +class Cursors: + operator: int + arize: int + + +def _cursors(pods: Pods) -> Cursors: + return Cursors(recorded_spans(pods.sinks.operator)[0], recorded_spans(pods.sinks.arize)[0]) + + +def _await_flushed(pods: Pods, cursors: Cursors, fence_key: str) -> None: + """Wait until every worker of both pods has sent the team's exporter everything it decided so far.""" + fences: Final = pods.fence(fence_key) + + def arrived() -> None: + delivered: Final = _delivered(_team_spans(pods, cursors.arize), fences) + assert {fence.call_id for fence in fences} <= delivered, "the unsampled team's calls have not arrived yet" + + until_asserted(arrived, seconds=30) + + +def _settled( + pods: Pods, + cursors: Cursors, + by_trace: Mapping[str, Sequence[Sent]], + fence_key: str, + holds: Callable[[tuple[bool, ...]], bool] = lambda kept: len(set(kept)) == 1, +) -> Verdicts: + """Per trace, which of its calls reached the team's space, once every worker has sent what it decided + and ``holds`` is true of every trace.""" + calls: Final = tuple(chain.from_iterable(by_trace.values())) + _await_logged(pods, cursors.operator, calls) + _await_flushed(pods, cursors, fence_key) + + def check() -> Verdicts: + delivered: Final = _delivered(_team_spans(pods, cursors.arize), calls) + verdicts: Final = { + trace_id: tuple(call.call_id in delivered for call in sent) for trace_id, sent in by_trace.items() + } + broken: Final = {trace_id: kept for trace_id, kept in verdicts.items() if not holds(kept)} + assert broken == {}, f"{len(broken)} of {len(verdicts)} traces were split (per call, kept): {broken}" + return verdicts + + return until_asserted(check, seconds=30) + + +def _trace_ids(cell: str, count: int = TRACES) -> tuple[str, ...]: + """Fixed RFC 4122 shaped trace ids, like the uuid4 hex a caller mints for ``traceparent``.""" + return tuple(uuid.uuid5(uuid.NAMESPACE_URL, f"team-sampling/{cell}/{index}").hex for index in range(count)) + + +def _kept(verdicts: Verdicts) -> int: + return sum(1 for kept in verdicts.values() if all(kept)) + + +@pytest.mark.parametrize( + ("variables", "content"), + [({"arize_success_sampling_rate": "0.5"}, "hello"), ({"arize_error_sampling_rate": "0.5"}, "sampling-fail")], + ids=["success-rate", "error-rate"], +) +def test_each_trace_reaches_the_team_space_whole_or_not_at_all( + pods: Pods, variables: Mapping[str, str], content: str +) -> None: + """The ticket's case: three calls per trace load balanced over the pods.""" + sampled: Final = pods.team_key({**CREDENTIALS, **variables}) + cursors: Final = _cursors(pods) + by_trace: Final = { + trace_id: tuple( + pods.send(pod, _call(sampled, trace_id, content=content)) for pod in (pods.first, pods.second, pods.first) + ) + for trace_id in _trace_ids("-".join(sorted(variables))) + } + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < TRACES, f"a rate of 0.5 kept {_kept(verdicts)} of {TRACES} traces" + + +@pytest.mark.parametrize("client", ["sync", "async"]) +def test_every_endpoint_follows_the_verdict_of_its_trace(pods: Pods, client: str) -> None: + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + plan: Final[tuple[tuple[OwnedProxy, Endpoint, bool], ...]] = ( + (pods.first, "chat", False), + (pods.second, "chat", True), + (pods.first, "responses", False), + (pods.second, "messages", True), + ) + + async def send_trace(trace_id: str) -> tuple[Sent, ...]: + clients: Final = { + id(pod): httpx.AsyncClient(base_url=str(pod.gateway.client.base_url), timeout=15, trust_env=False) + for pod in (pods.first, pods.second) + } + try: + return tuple( + await asyncio.gather( + *( + pods.send_async(clients[id(pod)], _call(sampled, trace_id, endpoint, stream)) + for pod, endpoint, stream in plan + ) + ) + ) + finally: + for opened in clients.values(): + await opened.aclose() + + by_trace: Final = { + trace_id: ( + tuple(pods.send(pod, _call(sampled, trace_id, endpoint, stream)) for pod, endpoint, stream in plan) + if client == "sync" + else asyncio.run(send_trace(trace_id)) + ) + for trace_id in _trace_ids(f"endpoints-{client}") + } + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < TRACES + + +def test_the_workers_of_one_pod_agree(pods: Pods) -> None: + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + by_trace: Final = { + trace_id: pods.on_each_worker(pods.second, lambda trace_id=trace_id: _call(sampled, trace_id)) + for trace_id in _trace_ids("workers") + } + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < TRACES + + +def test_a_replaced_worker_reaches_the_verdict_its_predecessor_reached(pods: Pods) -> None: + """A worker that restarts forgets the verdicts it remembered; the calls it serves next for the same + traces still answer the way the first ones did.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + fence_key: Final = pods.team_key(CREDENTIALS) + traces: Final = _trace_ids("replaced") + before: Final = { + trace_id: pods.on_each_worker(pods.second, lambda t=trace_id: _call(sampled, t)) for trace_id in traces + } + _await_logged(pods, cursors.operator, tuple(chain.from_iterable(before.values()))) + _await_flushed(pods, cursors, fence_key) + for victim in sorted(pods.second.worker_pids()): + pods.second.kill_worker(victim) + by_trace: Final = { + trace_id: before[trace_id] + pods.on_each_worker(pods.second, lambda t=trace_id: _call(sampled, t)) + for trace_id in traces + } + + verdicts: Final = _settled(pods, cursors, by_trace, fence_key) + + assert 0 < _kept(verdicts) < TRACES + + +def test_a_failed_call_is_kept_wherever_a_successful_call_of_its_trace_is(pods: Pods) -> None: + """One call of the trace fails on one pod and answers to the error rate, another succeeds on the other + pod and answers to the success rate: with the error rate above the success rate, no trace keeps its + successful call and drops its failed one.""" + sampled: Final = pods.team_key( + {**CREDENTIALS, "arize_success_sampling_rate": "0.3", "arize_error_sampling_rate": "0.7"} + ) + cursors: Final = _cursors(pods) + by_trace: Final = { + trace_id: ( + pods.send(pods.first, _call(sampled, trace_id, content="sampling-fail")), + pods.send(pods.second, _call(sampled, trace_id)), + ) + for trace_id in _trace_ids("mixed", 32) + } + + verdicts: Final = _settled( + pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept[0] or not kept[1] + ) + + assert any(kept == (True, False) for kept in verdicts.values()), "no trace sat between the two rates" + assert any(kept == (True, True) for kept in verdicts.values()), "no trace sat under both rates" + + +def test_a_failed_call_answers_to_the_error_rate_on_a_pod_that_dropped_its_trace(pods: Pods) -> None: + """A pod that already dropped a successful call of the trace still keeps a later failed call of it at an + unset error rate, as the pod that only served the failed call does.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.3"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("failed-after-dropped") + succeeded: Final = {trace_id: pods.send(pods.first, _call(sampled, trace_id)) for trace_id in traces} + _await_logged(pods, cursors.operator, tuple(succeeded.values())) + by_trace: Final = { + trace_id: ( + succeeded[trace_id], + pods.send(pods.first, _call(sampled, trace_id, content="sampling-fail")), + pods.send(pods.second, _call(sampled, trace_id, content="sampling-fail")), + ) + for trace_id in traces + } + + verdicts: Final = _settled( + pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept[1] and kept[2] + ) + + assert any(not kept[0] for kept in verdicts.values()), "no successful call was dropped before its trace failed" + + +def test_a_successful_call_answers_to_the_success_rate_on_a_pod_that_kept_its_failed_call(pods: Pods) -> None: + """A pod that already kept a failed call of the trace drops a later successful call of it at a zero + success rate, as the pod that only served the successful call does.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.0"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("succeeded-after-kept") + failed: Final = { + trace_id: pods.send(pods.first, _call(sampled, trace_id, content="sampling-fail")) for trace_id in traces + } + _await_logged(pods, cursors.operator, tuple(failed.values())) + by_trace: Final = { + trace_id: ( + failed[trace_id], + pods.send(pods.first, _call(sampled, trace_id)), + pods.send(pods.second, _call(sampled, trace_id)), + ) + for trace_id in traces + } + + _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept == (True, False, False)) + + +def test_concurrent_calls_of_one_trace_on_one_worker_each_answer_to_their_own_outcome(pods: Pods) -> None: + """A failed and a successful call of the trace in flight at once on the single worker of the first pod + are decided apart: the failed one is kept at an unset error rate, the successful one dropped at a zero + success rate.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.0"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("concurrent-mixed") + plan: Final = tuple(product(traces, ("sampling-fail", "hello"))) + + def send(job: tuple[str, str]) -> Sent: + return pods.send(pods.first, _call(sampled, job[0], content=job[1])) + + with ThreadPoolExecutor(max_workers=8) as pool: + sent: Final = tuple(pool.map(send, plan)) + by_trace: Final = {trace_id: tuple(call for call in sent if call.trace_id == trace_id) for trace_id in traces} + + _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept == (True, False)) + + +@pytest.mark.parametrize( + ("variables", "expected"), + [({"arize_success_sampling_rate": "1.0"}, TRACES), ({"arize_success_sampling_rate": "0.0"}, 0), ({}, TRACES)], + ids=["one", "zero", "unset"], +) +def test_a_rate_at_either_end_keeps_every_trace_or_none( + pods: Pods, variables: Mapping[str, str], expected: int +) -> None: + sampled: Final = pods.team_key({**CREDENTIALS, **variables}) + cursors: Final = _cursors(pods) + by_trace: Final = { + trace_id: tuple(pods.send(pod, _call(sampled, trace_id)) for pod in (pods.first, pods.second)) + for trace_id in _trace_ids(f"edge-{'-'.join(f'{k}={v}' for k, v in sorted(variables.items()))}") + } + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert _kept(verdicts) == expected + + +def test_a_request_without_a_traceparent_is_sampled_whole_at_the_rate(pods: Pods) -> None: + """With no caller trace the proxy starts one per request: each request's server and model spans travel + together and the rate still keeps some requests and drops others.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + sent: Final = tuple( + pods.send(pod, _call(sampled, None)) for _, pod in product(range(12), (pods.first, pods.second)) + ) + _await_logged(pods, cursors.operator, sent) + trace_of: Final = { + str(span["attributes"][CALL_ID]): span["trace_id"] + for span in recorded_spans(pods.sinks.operator, cursors.operator)[1] + if CALL_ID in span["attributes"] + } + fences: Final = pods.fence(pods.team_key(CREDENTIALS)) + + def check() -> Mapping[str, tuple[bool, bool]]: + spans: Final = _team_spans(pods, cursors.arize) + assert {fence.call_id for fence in fences} <= _delivered(spans, fences), ( + "the unsampled team's calls have not arrived yet" + ) + kinds: Final = { + call.call_id: ( + any(span["kind"] == SERVER for span in spans if span["trace_id"] == trace_of[call.call_id]), + any(OPERATION in span["attributes"] for span in spans if span["trace_id"] == trace_of[call.call_id]), + ) + for call in sent + } + partial: Final = {call_id: kept for call_id, kept in kinds.items() if kept[0] != kept[1]} + assert partial == {}, f"requests reached the space without their whole tree (server, model): {partial}" + return kinds + + kept: Final = sum(1 for server, _ in until_asserted(check, seconds=30).values() if server) + assert 0 < kept < len(sent), f"a rate of 0.5 kept {kept} of {len(sent)} requests" + + +def test_a_key_level_destination_follows_the_verdict_of_its_trace(pods: Pods) -> None: + team: Final = pods.scenario.team() + sampled: Final = pods.scenario.key( + team_id=team, + metadata={ + "logging": [ + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {**CREDENTIALS, "arize_success_sampling_rate": "0.5"}, + } + ] + }, + ) + cursors: Final = _cursors(pods) + by_trace: Final = { + trace_id: tuple(pods.send(pod, _call(sampled, trace_id)) for pod in (pods.first, pods.second, pods.first)) + for trace_id in _trace_ids("key-level") + } + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < TRACES + + +def test_a_concurrent_burst_keeps_every_trace_whole(pods: Pods) -> None: + """Four calls per trace, all in flight at once across both pods.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("burst", 24) + plan: Final = tuple(product(traces, (pods.first, pods.second, pods.first, pods.second))) + + def send(job: tuple[str, OwnedProxy]) -> Sent: + return pods.send(job[1], _call(sampled, job[0])) + + with ThreadPoolExecutor(max_workers=16) as pool: + sent: Final = list(pool.map(send, plan)) + by_trace: Final = {trace_id: tuple(call for call in sent if call.trace_id == trace_id) for trace_id in traces} + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < len(traces) + + +def test_a_stalled_team_space_during_a_burst_keeps_serving_and_each_trace_whole(pods: Pods) -> None: + """The team's Arize endpoint stops answering while a burst of calls runs over both pods: every call is + still served, and once the endpoint answers again each trace arrives whole or not at all.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("stalled") + plan: Final = tuple(product(traces, (pods.first, pods.second))) + + def send(job: tuple[str, OwnedProxy]) -> Sent: + return pods.send(job[1], _call(sampled, job[0])) + + configure_sink(pods.sinks.arize, paused=True) + try: + with ThreadPoolExecutor(max_workers=8) as pool: + sent: Final = tuple(pool.map(send, plan)) + finally: + configure_sink(pods.sinks.arize, paused=False) + by_trace: Final = {trace_id: tuple(call for call in sent if call.trace_id == trace_id) for trace_id in traces} + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < len(traces) + + +def _write_config(directory: Path, sinks: SpanSinks, provider: Wire) -> Path: + shared: Final = object_value( + JSON.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + ) + config: Final = { + **shared, + "model_list": [ + *list_value(shared.get("model_list") or []), + { + "model_name": MODEL, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "integration-provider-key", + "api_base": provider.url + "/v1", + }, + }, + ], + "litellm_settings": { + **object_value(shared.get("litellm_settings") or {}), + "callbacks": ["langfuse_otel"], + "otel_tenant_destination_mode": "additive", + "provider_url_destination_allowed_hosts": [urlparse(sinks.operator).netloc, urlparse(sinks.arize).netloc], + }, + "router_settings": {**object_value(shared.get("router_settings") or {}), "num_retries": 0}, + "general_settings": {**object_value(shared.get("general_settings") or {}), "user_api_key_cache_ttl": 2}, + } + path: Final = directory / "team-sampling.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@contextmanager +def _two_pods(provider: Wire, sinks: SpanSinks, directory: Path) -> Generator[Pods]: + environment: Final = { + "LITELLM_OTEL_V2": "1", + "OTEL_BSP_SCHEDULE_DELAY": "300", + "MAPPER_NAMES": "genai,openinference", + "LANGFUSE_HOST": sinks.operator, + "LANGFUSE_PUBLIC_KEY": "pk-lf-operator", + "LANGFUSE_SECRET_KEY": "sk-lf-operator", + "ARIZE_SPACE_KEY": "space-operator", + "ARIZE_API_KEY": "arize-operator", + "ARIZE_HTTP_ENDPOINT": sinks.arize + "/v1/traces", + } + removed: Final = ("LITELLM_OTEL_TENANT_DESTINATION_MODE", "OTEL_TRACES_SAMPLER", "OTEL_TRACES_SAMPLER_ARG") + config: Final = _write_config(directory, sinks, provider) + for pod in ("first", "second"): + (directory / pod).mkdir() + with gateway_from_environment() as shared, scratch_database("team_sampling") as database_url: + overrides: Final = {**environment, "DATABASE_URL": database_url} + with ( + owned_proxy_process( + shared, directory / "first", overrides, config=config, remove_environment=removed + ) as first, + owned_proxy_process( + shared, + directory / "second", + overrides, + config=config, + remove_environment=removed, + workers=2, + healthcheck=WorkerHealthcheck.LENIENT, + ) as second, + replace(first.gateway, database_url=database_url).scenario() as scenario, + ): + eventually(second.worker_pids, lambda pids: len(pids) == 2, seconds=120) + yield Pods(first, second, scenario, sinks) + + +@pytest.fixture(scope="module") +def provider() -> Iterator[Wire]: + with wire_server(_upstream) as wire: + yield wire + + +@pytest.fixture(scope="module") +def pods(provider: Wire, audit_sinks: SpanSinks, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Pods]: + with _two_pods(provider, audit_sinks, tmp_path_factory.mktemp("team-sampling")) as started: + yield started diff --git a/tests/integration/observability/test_passthrough_guardrail_scan_text.py b/tests/integration/observability/test_passthrough_guardrail_scan_text.py new file mode 100644 index 00000000000..7437f696d4c --- /dev/null +++ b/tests/integration/observability/test_passthrough_guardrail_scan_text.py @@ -0,0 +1,153 @@ +import json +import uuid +from collections.abc import Iterator, Mapping +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest +import yaml +from integration._support.anthropic_sse import message_json +from integration._support.client import Gateway +from integration._support.openai_wire import chat_reply +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +GENERIC: Final = "passthrough-scan-generic" +AIRS: Final = "passthrough-scan-airs" +OPENAI_ROUTE: Final = "/openai/chat/completions" +ANTHROPIC_ROUTE: Final = "/anthropic/v1/messages" +ROUTE_BODIES: Final[Mapping[str, dict[str, JsonValue]]] = MappingProxyType( + { + OPENAI_ROUTE: {"model": "gpt-5.6", "messages": [{"role": "user", "content": "Say hi in one word"}]}, + ANTHROPIC_ROUTE: { + "model": "claude-sonnet-5-5", + "max_tokens": 16, + "messages": [{"role": "user", "content": "Say hi in one word"}], + }, + } +) +_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" +_JSON: Final = TypeAdapter(dict[str, JsonValue]) + + +def _generic_verdict(_request: Request) -> Reply: + return Reply(body=json.dumps({"action": "NONE"}).encode()) + + +def _airs_verdict(_request: Request) -> Reply: + return Reply(body=json.dumps({"action": "allow", "category": "benign", "scan_id": "s", "report_id": "r"}).encode()) + + +def _openai_reply(_request: Request) -> Reply: + return chat_reply("chatcmpl-passthrough-scan", "gpt-5.6", "Hi", stream=False) + + +def _anthropic_reply(_request: Request) -> Reply: + return Reply(body=message_json("msg_passthrough_scan", "claude-sonnet-5-5", "Hi")) + + +def _parsed_texts(texts: JsonValue, vendor_body: dict[str, JsonValue]) -> list[JsonValue]: + assert isinstance(texts, list), vendor_body + return [json.loads(text) if isinstance(text, str) else text for text in texts] + + +def _scanned_payloads(guardrail: str, vendor_body: dict[str, JsonValue]) -> list[JsonValue]: + if guardrail == GENERIC: + return _parsed_texts(vendor_body["texts"], vendor_body) + contents: Final = vendor_body["contents"] + assert isinstance(contents, list), vendor_body + return _parsed_texts([content.get("prompt") for content in contents if isinstance(content, dict)], vendor_body) + + +@dataclass(frozen=True, slots=True) +class PassThroughRig: + owned: OwnedProxy + vendors: Mapping[str, Wire] + upstreams: Mapping[str, Wire] + + +@pytest.fixture(scope="module") +def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[PassThroughRig]: + directory: Final = tmp_path_factory.mktemp("passthrough-guardrail-scan") + with ExitStack() as stack: + vendors: Final = MappingProxyType( + { + GENERIC: stack.enter_context(wire_server(_generic_verdict)), + AIRS: stack.enter_context(wire_server(_airs_verdict)), + } + ) + upstreams: Final = MappingProxyType( + { + OPENAI_ROUTE: stack.enter_context(wire_server(_openai_reply)), + ANTHROPIC_ROUTE: stack.enter_context(wire_server(_anthropic_reply)), + } + ) + config: Final = _JSON.validate_python(yaml.safe_load(_CONFIG.read_text())) + path: Final = directory / "passthrough-guardrail-scan.yaml" + path.write_text( + yaml.safe_dump( + { + **config, + "guardrails": [ + { + "guardrail_name": GENERIC, + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "api_base": vendors[GENERIC].url, + "api_key": "synthetic-guardrail-key", + }, + }, + { + "guardrail_name": AIRS, + "litellm_params": { + "guardrail": "panw_prisma_airs", + "mode": "pre_call", + "default_on": False, + "api_base": vendors[AIRS].url, + "api_key": "synthetic-airs-key", + "profile_name": "integration", + }, + }, + ], + } + ) + ) + environment: Final = { + "OPENAI_API_BASE": upstreams[OPENAI_ROUTE].url, + "OPENAI_API_KEY": "synthetic-openai-key", + "ANTHROPIC_API_BASE": upstreams[ANTHROPIC_ROUTE].url, + "ANTHROPIC_API_KEY": "synthetic-anthropic-key", + } + owned: Final = stack.enter_context(owned_proxy_process(session_gateway, directory, environment, config=path)) + yield PassThroughRig(owned, vendors, upstreams) + + +@pytest.mark.parametrize("guardrail", [GENERIC, AIRS]) +@pytest.mark.parametrize("route", [OPENAI_ROUTE, ANTHROPIC_ROUTE]) +def test_pass_through_guardrail_scans_the_client_body_without_the_callers_key_metadata( + rig: PassThroughRig, route: str, guardrail: str +) -> None: + gateway: Final = rig.owned.gateway + with gateway.scenario() as scenario: + team: Final = scenario.team() + user: Final = scenario.user( + user_email=f"passthrough-scan-{uuid.uuid4().hex}@example.com", user_role="internal_user", teams=[team] + ) + key: Final = scenario.key(user_id=user, team_id=team) + body: Final = {**ROUTE_BODIES[route], "guardrails": [guardrail]} + + response: Final = gateway.request("POST", route, body, key=key) + + assert response.status_code == 200, response.text + scanned: Final = [ + _scanned_payloads(guardrail, _JSON.validate_json(request.body)) + for request in rig.vendors[guardrail].drain() + ] + assert scanned == [[body]], scanned + assert len(rig.upstreams[route].drain()) == 1 diff --git a/tests/integration/observability/test_prometheus_provider_auth_failure.py b/tests/integration/observability/test_prometheus_provider_auth_failure.py new file mode 100644 index 00000000000..13d264ca6b0 --- /dev/null +++ b/tests/integration/observability/test_prometheus_provider_auth_failure.py @@ -0,0 +1,1074 @@ +from __future__ import annotations + +import asyncio +import json +import threading +import uuid +from collections import Counter +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import httpx +import openai +import pytest +import yaml +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment +from integration._support.database import write_rows +from integration._support.mcp import ENTRY_POINTS, McpCaller, McpPeer, echo_tool, register_mcp, scripted_peer +from integration._support.process import owned_proxy +from integration._support.prometheus_series import Sample, scrape +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +RUN: Final = uuid.uuid4().hex[:8] +REJECTS: Final = "provider-rejects-credential" +HELD_REJECTS: Final = "provider-rejects-credential-when-released" +KEY_NOT_VALID: Final = "provider-key-not-valid" +RATE_LIMITS: Final = "provider-rate-limits" +RECOVERS: Final = "provider-rejects-once" +HEALTHY: Final = "provider-healthy" +AGENT_365_GUARDRAIL: Final = f"agent-365-{RUN}" +CALLER_OAUTH_TOKEN: Final = f"sk-ant-oat01-caller-{RUN}" +REVOKED_GATEWAY_KEY: Final = f"revoked-gateway-key-{RUN}" +UNKNOWN_ORGANIZATION: Final = f"org-unknown-{RUN}" +REVOKED_STORED_KEY: Final = f"sk-revoked-stored-credential-{RUN}" +AZURE_API_VERSION: Final = "2024-02-01" +CALLER_AZURE_CELLS: Final = ( + ("chat", False), + ("embeddings", False), + ("responses", False), + ("messages", False), + ("responses", True), +) +CALLER_CREDENTIAL_CELLS: Final = ( + ("caller-watsonx-token", "token"), + ("caller-gigachat-access-token", "gigachat_access_token"), +) +TOKEN_EXCHANGE_CELLS: Final = ("token-exchange", "token-exchange-credential") +KEYLESS_CELLS: Final = ("keyless", "messages-keyless") +TOKEN_EXCHANGE_SETTINGS: Final = { + "token_exchange_endpoint": "https://login.microsoftonline.com/00000000-0000-4000-8000-0000000c0b01/oauth2/v2.0/token", + "client_id": "synthetic-obo-client-id", + "client_secret": "synthetic-obo-client-secret", +} + +FAILURES: Final = "litellm_deployment_failure_responses_total" +TOTALS: Final = "litellm_deployment_total_requests_total" +LLM_FAILED: Final = "litellm_llm_api_failed_requests_metric_total" +STATE: Final = "litellm_deployment_state" +PROXY_FAILED: Final = "litellm_proxy_failed_requests_metric_total" +SUCCESSES: Final = "litellm_deployment_success_responses_total" +DEPLOYMENT_SERIES: Final = (FAILURES, TOTALS, LLM_FAILED, STATE) + +ENDPOINT_CELLS: Final = ( + "chat", + "chat-stream", + "messages", + "messages-stream", + "responses", + "responses-stream", + "embeddings", +) +REJECTED_CELLS: Final = ( + *ENDPOINT_CELLS, + "sdk-sync", + "sdk-async", + "repeated", + "fallback", + "burst", + "caller-api-key", + "caller-extra-headers", + "caller-headers", + "caller-provider-specific-header", + "plain-extra-headers", + "plain-headers", + "plain-provider-specific-header", + "forwarded-provider-header", + "forwarded-plain-header", + "forwarded-user-information", +) +GATED_CELLS: Final = ("invalid-key", "missing-key", "expired-key", "blocked-key", "no-model-access") +CALLER_SCOPE_CELLS: Final = ("forwarded-gateway-key", "caller-organization", "forwarded-organization") +HEALTHY_CELLS: Final = (*GATED_CELLS, *CALLER_SCOPE_CELLS, "healthy", "key-rpm-limit") +GUARDRAIL_ROWS: Final = ( + "SELECT COALESCE(jsonb_path_query_first(metadata, " + "'$.guardrail_information[*] ? (@.guardrail_name == $name).guardrail_status', " + "jsonb_build_object('name', %s::text)) #>> '{}', 'none') AS status " + 'FROM "LiteLLM_SpendLogs" WHERE api_key = %s AND call_type = %s' +) + + +def name(cell: str) -> str: + return f"prometheus-{cell}-{RUN}" + + +def identity(cell: str) -> str: + return f"{name(cell)}-deployment" + + +def _error(status: int, message: str, kind: str, code: str) -> Reply: + return Reply(status=status, body=json.dumps({"error": {"message": message, "type": kind, "code": code}}).encode()) + + +def _completion() -> Reply: + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + ).encode() + ) + + +class Provider: + """Answers by upstream model name. `HELD_REJECTS` waits for `release` before rejecting the credential.""" + + def __init__(self, release: threading.Event) -> None: + self._lock: Final = threading.Lock() + self._seen: Final[Counter[str]] = Counter() + self._release: Final = release + + def __call__(self, request: Request) -> Reply: + sent: Final = json.loads(request.body) + model: Final = str(sent.get("model") or sent.get("model_id") or request.target.split("/")[2]) + if request.headers.get("x-portkey-api-key") == REVOKED_GATEWAY_KEY: + return _error(401, "Invalid gateway API key", "invalid_request_error", "invalid_api_key") + if request.headers.get("authorization") == f"Bearer {REVOKED_STORED_KEY}": + return _error(401, "Incorrect API key provided: sk-revok*****", "invalid_request_error", "invalid_api_key") + if request.headers.get("openai-organization") == UNKNOWN_ORGANIZATION: + return _error( + 401, + "OpenAI-Organization header should match organization for API key", + "invalid_request_error", + "mismatched_organization", + ) + with self._lock: + self._seen[model] += 1 + attempt: Final = self._seen[model] + if model == HELD_REJECTS: + assert self._release.wait(timeout=30), "held provider reply was never released" + if model in (REJECTS, HELD_REJECTS) or (model == RECOVERS and attempt == 1): + return _error(401, "Incorrect API key provided: sk-rotat*****", "invalid_request_error", "invalid_api_key") + if model == KEY_NOT_VALID: + return _error(400, "API key not valid. Please pass a valid API key.", "invalid_request_error", "400") + if model == RATE_LIMITS: + return _error(429, "Rate limit reached for requests", "rate_limit_error", "rate_limit_exceeded") + return _completion() + + +def deployment(cell: str, upstream_model: str, api_base: str) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": { + "model": f"openai/{upstream_model}", + "api_key": f"sk-deployment-credential-{cell}", + "api_base": api_base, + }, + "model_info": {"id": identity(cell)}, + } + + +def anthropic_deployment(cell: str, api_base: str) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": { + "model": f"anthropic/{REJECTS}", + "api_key": f"sk-ant-api03-deployment-credential-{cell}", + "api_base": api_base, + }, + "model_info": {"id": identity(cell)}, + } + + +def bedrock_deployment(cell: str, opt_in: dict[str, JsonValue]) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": { + "model": f"bedrock/converse/{REJECTS}", + "aws_access_key_id": "AKIADEPLOYMENTCREDENTIAL", + "aws_secret_access_key": f"deployment-credential-{cell}", + "aws_region_name": "us-east-1", + **opt_in, + }, + "model_info": {"id": identity(cell)}, + } + + +def keyless_anthropic_deployment(cell: str, api_base: str) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": {"model": f"anthropic/{REJECTS}", "api_base": api_base}, + "model_info": {"id": identity(cell)}, + } + + +def azure_deployment(cell: str, api_base: str, credential: dict[str, JsonValue]) -> dict[str, JsonValue]: + return { + "model_name": name(cell), + "litellm_params": { + "model": f"azure/{REJECTS}", + "api_base": api_base, + "api_version": AZURE_API_VERSION, + **credential, + }, + "model_info": {"id": identity(cell)}, + } + + +def caller_credential_deployment(cell: str, field: str, api_base: str) -> dict[str, JsonValue]: + provider: Final = "watsonx" if field == "token" else "gigachat" + project: Final[dict[str, JsonValue]] = {"project_id": f"project-{RUN}"} if provider == "watsonx" else {} + return { + "model_name": name(cell), + "litellm_params": { + "model": f"{provider}/{REJECTS}", + "api_base": api_base, + field: f"deployment-credential-{cell}", + **project, + }, + "model_info": {"id": identity(cell)}, + } + + +def token_exchange_deployment(cell: str) -> dict[str, JsonValue]: + settings: Final[dict[str, JsonValue]] = ( + {"litellm_credential_name": f"m365-obo-{RUN}"} if cell.endswith("-credential") else {**TOKEN_EXCHANGE_SETTINGS} + ) + return { + "model_name": name(cell), + "litellm_params": {"model": "microsoft_365_copilot/chat", **settings}, + "model_info": {"id": identity(cell)}, + } + + +def caller_azure_cell(opt_in: str, route: str, inside_extra_body: bool) -> str: + return f"{opt_in}-caller-azure-token-{route}{'-extra-body' if inside_extra_body else ''}" + + +def config(directory: Path, upstream: Wire) -> Path: + api_base: Final = f"{upstream.url}/v1" + configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration["model_list"] = [ + *configuration["model_list"], + *(deployment(cell, REJECTS, api_base) for cell in REJECTED_CELLS), + *(deployment(cell, HEALTHY, api_base) for cell in HEALTHY_CELLS), + deployment("held", HELD_REJECTS, api_base), + deployment("key-not-valid", KEY_NOT_VALID, api_base), + deployment("rate-limited", RATE_LIMITS, api_base), + deployment("recovers", RECOVERS, api_base), + anthropic_deployment("anthropic-own-key", upstream.url), + anthropic_deployment("anthropic-sdk-own-key", upstream.url), + anthropic_deployment("caller-oauth", upstream.url), + *(keyless_anthropic_deployment(cell, upstream.url) for cell in KEYLESS_CELLS), + bedrock_deployment( + "deployment-caller-bedrock-endpoint", + {"configurable_clientside_auth_params": ["aws_bedrock_runtime_endpoint"]}, + ), + azure_deployment("azure-own-token", upstream.url, {"azure_ad_token": "deployment-entra-token-azure-own-token"}), + *( + azure_deployment( + caller_azure_cell("deployment", route, inside_extra_body), + upstream.url, + {"configurable_clientside_auth_params": ["azure_ad_token"]}, + ) + for route, inside_extra_body in CALLER_AZURE_CELLS + ), + *(caller_credential_deployment(cell, field, upstream.url) for cell, field in CALLER_CREDENTIAL_CELLS), + *(token_exchange_deployment(cell) for cell in TOKEN_EXCHANGE_CELLS), + { + "model_name": name("caller-api-base"), + "litellm_params": { + "model": f"openai/{REJECTS}", + "api_key": "sk-deployment-credential-caller-api-base", + "api_base": api_base, + "configurable_clientside_auth_params": ["api_base"], + }, + "model_info": {"id": identity("caller-api-base")}, + }, + { + "model_name": name("caller-credential"), + "litellm_params": { + "model": "github_copilot/gpt-4o", + "litellm_credential_name": f"copilot-per-user-{RUN}", + "github_copilot_auth_type": "per_user_oauth", + }, + "model_info": {"id": identity("caller-credential")}, + }, + { + "model_name": name("caller-credential-name"), + "litellm_params": { + "model": f"openai/{HEALTHY}", + "api_base": api_base, + "litellm_credential_name": f"deployment-key-{RUN}", + }, + "model_info": {"id": identity("caller-credential-name")}, + }, + ] + configuration["credential_list"] = [ + *configuration.get("credential_list", []), + {"credential_name": f"m365-obo-{RUN}", "credential_values": {**TOKEN_EXCHANGE_SETTINGS}, "credential_info": {}}, + { + "credential_name": f"deployment-key-{RUN}", + "credential_values": {"api_key": "sk-deployment-credential-caller-credential-name"}, + "credential_info": {}, + }, + { + "credential_name": f"revoked-key-{RUN}", + "credential_values": {"api_key": REVOKED_STORED_KEY}, + "credential_info": {}, + }, + ] + configuration["guardrails"] = [ + { + "guardrail_name": AGENT_365_GUARDRAIL, + "litellm_params": { + "guardrail": "agent_365", + "mode": "pre_mcp_call", + "default_on": True, + "tenant_id": "00000000-0000-4000-8000-0000000a3650", + "client_id": "synthetic-client-id", + "client_secret": "synthetic-client-secret", + }, + } + ] + configuration["litellm_settings"]["callbacks"] = ["prometheus"] + configuration["litellm_settings"]["model_group_settings"] = { + "forward_client_headers_to_llm_api": [ + name("forwarded-provider-header"), + name("forwarded-plain-header"), + name("forwarded-gateway-key"), + name("forwarded-user-information"), + ] + } + configuration["litellm_settings"]["add_user_information_to_llm_headers"] = True + configuration["general_settings"]["forward_llm_provider_auth_headers"] = True + configuration["general_settings"]["forward_openai_org_id"] = True + configuration.setdefault("router_settings", {}).update({"num_retries": 0, "disable_cooldowns": True}) + path: Final = directory / "prometheus-provider-auth-failure.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + upstream: Wire + + @property + def api_base(self) -> str: + return f"{self.upstream.url}/v1" + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("prometheus-provider-auth-failure") + released: Final = threading.Event() + released.set() + with ( + wire_server(Provider(released)) as upstream, + gateway_from_environment() as shared, + owned_proxy( + shared, + directory, + {"PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "2"}, + config=config(directory, upstream), + remove_environment=("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN"), + ) as owned, + ): + yield Rig(owned, upstream) + + +def opted_in_config(directory: Path, upstream: Wire) -> Path: + configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration["model_list"] = [ + *configuration["model_list"], + *( + azure_deployment(caller_azure_cell("proxy-wide", route, inside_extra_body), upstream.url, {}) + for route, inside_extra_body in CALLER_AZURE_CELLS + ), + bedrock_deployment("proxy-wide-caller-bedrock-endpoint", {}), + ] + configuration["general_settings"]["allow_client_side_credentials"] = True + configuration["litellm_settings"]["callbacks"] = ["prometheus"] + configuration.setdefault("router_settings", {}).update({"num_retries": 0, "disable_cooldowns": True}) + path: Final = directory / "prometheus-caller-credentials-opted-in.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +@pytest.fixture(scope="module") +def opted_in(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("prometheus-caller-credentials-opted-in") + released: Final = threading.Event() + released.set() + with ( + wire_server(Provider(released)) as upstream, + gateway_from_environment() as shared, + owned_proxy(shared, directory, {}, config=opted_in_config(directory, upstream)) as owned, + ): + yield Rig(owned, upstream) + + +def series(samples: Sequence[Sample], metric: str, **labels: str) -> tuple[Sample, ...]: + return tuple( + sample + for sample in samples + if sample.name == metric and all(sample.labels.get(label) == value for label, value in labels.items()) + ) + + +def values(samples: Sequence[Sample], metric: str, **labels: str) -> list[float]: + return [sample.value for sample in series(samples, metric, **labels)] + + +def until(gateway: Gateway, metric: str, **labels: str) -> tuple[Sample, ...]: + return eventually(lambda: scrape(gateway), lambda samples: bool(series(samples, metric, **labels)), 30) + + +def messages(cell: str) -> list[JsonValue]: + return [{"role": "user", "content": f"{cell} {uuid.uuid4().hex}"}] + + +def send(gateway: Gateway, cell: str, model: str | None = None, key: str | None = None) -> httpx.Response: + asked: Final = model or name(cell) + stream: Final = cell.endswith("-stream") + if cell.startswith("messages"): + anthropic: Final = {"model": asked, "max_tokens": 16, "messages": messages(cell), "stream": stream} + return gateway.request("POST", "/v1/messages", anthropic, key=key) + if cell.startswith("responses"): + return gateway.request("POST", "/v1/responses", {"model": asked, "input": cell, "stream": stream}, key=key) + if cell == "embeddings": + return gateway.request("POST", "/v1/embeddings", {"model": asked, "input": cell}, key=key) + return gateway.request( + "POST", "/v1/chat/completions", {"model": asked, "messages": messages(cell), "stream": stream}, key=key + ) + + +def assert_reached_with_the_deployment_credential(rig: Rig, cell: str, attempts: int = 1) -> None: + reached: Final = rig.upstream.drain() + assert [request.headers["authorization"] for request in reached] == [ + f"Bearer sk-deployment-credential-{cell}" + ] * attempts, [request.target for request in reached] + + +def recorded(samples: Sequence[Sample], **labels: str) -> list[tuple[str, dict[str, str], float]]: + return [(sample.name, dict(sample.labels), sample.value) for sample in series_of(samples, **labels)] + + +def series_of(samples: Sequence[Sample], **labels: str) -> tuple[Sample, ...]: + return tuple( + sample + for sample in samples + if sample.name.startswith("litellm_") + and all(sample.labels.get(label) == value for label, value in labels.items()) + ) + + +def assert_kept_out_of_the_deployment_metrics(rig: Rig, cell: str) -> None: + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name(cell)) + assert {metric: values(samples, metric, model_id=identity(cell)) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + }, recorded(samples, model_id=identity(cell)) + + +def assert_recorded_as_a_deployment_failure(rig: Rig, cell: str, status: str, count: float = 1.0) -> None: + samples: Final = until(rig.gateway, FAILURES, model_id=identity(cell)) + observed: Final = recorded(samples, model_id=identity(cell)) + assert values(samples, FAILURES, model_id=identity(cell), exception_status=status) == [count], observed + assert values(samples, TOTALS, model_id=identity(cell)) == [count], observed + assert values(samples, LLM_FAILED, model_id=identity(cell)) == [count], observed + assert values(samples, STATE, model_id=identity(cell)) == [1.0], observed + + +@pytest.mark.parametrize("cell", ENDPOINT_CELLS) +def test_a_provider_401_on_every_endpoint_counts_as_a_deployment_failure_and_a_partial_outage( + rig: Rig, cell: str +) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, cell) + assert response.status_code == 401, response.text + assert_reached_with_the_deployment_credential(rig, cell) + assert_recorded_as_a_deployment_failure(rig, cell, "401") + + +def test_a_provider_401_seen_by_the_sync_openai_client_is_recorded(rig: Rig) -> None: + rig.upstream.drain() + with ( + openai.OpenAI( + base_url=f"{rig.gateway.client.base_url}/v1", + api_key=rig.gateway.key, + max_retries=0, + http_client=httpx.Client(trust_env=False), + ) as client, + pytest.raises(openai.AuthenticationError, match="Incorrect API key provided"), + ): + client.chat.completions.create(model=name("sdk-sync"), messages=[{"role": "user", "content": "sync"}]) + assert_reached_with_the_deployment_credential(rig, "sdk-sync") + assert_recorded_as_a_deployment_failure(rig, "sdk-sync", "401") + + +def test_a_provider_401_seen_by_the_async_openai_client_is_recorded(rig: Rig) -> None: + async def ask() -> None: + async with openai.AsyncOpenAI( + base_url=f"{rig.gateway.client.base_url}/v1", + api_key=rig.gateway.key, + max_retries=0, + http_client=httpx.AsyncClient(trust_env=False), + ) as client: + await client.chat.completions.create(model=name("sdk-async"), messages=[{"role": "user", "content": "a"}]) + + rig.upstream.drain() + with pytest.raises(openai.AuthenticationError, match="Incorrect API key provided"): + asyncio.run(ask()) + assert_reached_with_the_deployment_credential(rig, "sdk-async") + assert_recorded_as_a_deployment_failure(rig, "sdk-async", "401") + + +def test_each_repeated_provider_401_is_counted_once(rig: Rig) -> None: + rig.upstream.drain() + assert [send(rig.gateway, "repeated").status_code for _ in range(3)] == [401, 401, 401] + assert_reached_with_the_deployment_credential(rig, "repeated", attempts=3) + assert_recorded_as_a_deployment_failure(rig, "repeated", "401", count=3.0) + + +def test_a_provider_401_rescued_by_a_fallback_still_counts_against_the_rejected_deployment(rig: Rig) -> None: + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name("fallback"), "messages": messages("fallback"), "fallbacks": [name("healthy")]}, + ) + assert response.status_code == 200, response.text + assert [request.headers["authorization"] for request in rig.upstream.drain()] == [ + "Bearer sk-deployment-credential-fallback", + "Bearer sk-deployment-credential-healthy", + ] + assert_recorded_as_a_deployment_failure(rig, "fallback", "401") + rescued: Final = eventually( + lambda: scrape(rig.gateway), lambda samples: values(samples, SUCCESSES, model_id=identity("healthy")) == [1.0] + ) + assert values(rescued, FAILURES, model_id=identity("healthy")) == [] + + +def test_a_provider_auth_message_on_a_non_401_status_is_recorded(rig: Rig) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, "key-not-valid") + assert response.status_code == 400, response.text + assert_reached_with_the_deployment_credential(rig, "key-not-valid") + assert_recorded_as_a_deployment_failure(rig, "key-not-valid", "400") + + +def test_a_provider_429_is_recorded_as_before(rig: Rig) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, "rate-limited") + assert response.status_code == 429, response.text + assert_reached_with_the_deployment_credential(rig, "rate-limited") + assert_recorded_as_a_deployment_failure(rig, "rate-limited", "429") + + +def test_a_deployment_marked_down_by_a_provider_401_is_healthy_again_after_a_success(rig: Rig) -> None: + rig.upstream.drain() + assert send(rig.gateway, "recovers").status_code == 401 + assert_recorded_as_a_deployment_failure(rig, "recovers", "401") + recovered: Final = send(rig.gateway, "recovers") + assert recovered.status_code == 200, recovered.text + assert len(rig.upstream.drain()) == 2 + eventually( + lambda: scrape(rig.gateway), + lambda samples: values(samples, STATE, model_id=identity("recovers"), api_base=rig.api_base) == [0.0], + ) + + +def test_a_per_caller_credential_rejection_stays_out_of_the_deployment_metrics(rig: Rig) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, "caller-credential") + assert response.status_code == 401, response.text + assert "Connect GitHub Copilot" in response.text, response.text + assert rig.upstream.drain() == () + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name("caller-credential")) + assert values(samples, FAILURES, model_id=identity("caller-credential")) == [] + assert values(samples, TOTALS, model_id=identity("caller-credential")) == [] + assert values(samples, LLM_FAILED, model_id=identity("caller-credential")) == [] + assert values(samples, STATE, model_id=identity("caller-credential")) == [] + + +def deployment_series(samples: Sequence[Sample]) -> frozenset[tuple[str, tuple[tuple[str, str], ...]]]: + return frozenset((sample.name, sample.identity()) for sample in samples if sample.name in DEPLOYMENT_SERIES) + + +def test_a_provider_401_on_an_api_key_the_caller_sent_stays_out_of_the_deployment_metrics(rig: Rig) -> None: + caller_key: Final = f"sk-caller-supplied-{uuid.uuid4().hex}" + before: Final = deployment_series(scrape(rig.gateway)) + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name("caller-api-key"), "messages": messages("caller-api-key"), "api_key": caller_key}, + ) + assert response.status_code == 401, response.text + assert [request.headers["authorization"] for request in rig.upstream.drain()] == [f"Bearer {caller_key}"] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name("caller-api-key")) + assert deployment_series(samples) - before == frozenset(), recorded(samples, requested_model=name("caller-api-key")) + + +def test_a_provider_401_on_a_stored_credential_the_caller_names_leaves_a_healthy_deployment_healthy(rig: Rig) -> None: + cell: Final = "caller-credential-name" + rig.upstream.drain() + assert send(rig.gateway, cell).status_code == 200 + assert_reached_with_the_deployment_credential(rig, cell) + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name(cell), "messages": messages(cell), "litellm_credential_name": f"revoked-key-{RUN}"}, + ) + assert response.status_code == 401, response.text + assert [request.headers["authorization"] for request in rig.upstream.drain()] == [f"Bearer {REVOKED_STORED_KEY}"] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name(cell)) + observed: Final = recorded(samples, model_id=identity(cell)) + assert values(samples, FAILURES, model_id=identity(cell)) == [], observed + assert values(samples, LLM_FAILED, model_id=identity(cell)) == [], observed + assert values(samples, TOTALS, model_id=identity(cell)) == [1.0], observed + assert values(samples, STATE, model_id=identity(cell)) == [0.0], observed + + +def anthropic_messages(gateway: Gateway, cell: str, key: str, headers: dict[str, str]) -> httpx.Response: + body: Final[dict[str, JsonValue]] = {"model": name(cell), "max_tokens": 16, "messages": messages(cell)} + return gateway.request("POST", "/v1/messages", body, key=key, headers=headers) + + +def caller_header(field: str, header: str, value: str) -> JsonValue: + if field == "provider_specific_header": + return {"custom_llm_provider": "openai", "extra_headers": {header: value}} + return {header: value} + + +@pytest.mark.parametrize( + "cell,field", + [ + ("caller-extra-headers", "extra_headers"), + ("caller-headers", "headers"), + ("caller-provider-specific-header", "provider_specific_header"), + ], +) +def test_a_provider_401_on_an_authorization_header_the_caller_sends_stays_out_of_the_deployment_metrics( + rig: Rig, cell: str, field: str +) -> None: + caller_value: Final = f"Bearer sk-{cell}-{uuid.uuid4().hex}" + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name(cell), "messages": messages(cell), field: caller_header(field, "Authorization", caller_value)}, + ) + assert response.status_code == 401, response.text + assert [request.headers["authorization"] for request in rig.upstream.drain()] == [caller_value] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name(cell)) + observed: Final = recorded(samples, model_id=identity(cell)) + assert values(samples, FAILURES, model_id=identity(cell)) == [], observed + assert values(samples, TOTALS, model_id=identity(cell)) == [], observed + assert values(samples, LLM_FAILED, model_id=identity(cell)) == [], observed + assert values(samples, STATE, model_id=identity(cell)) == [], observed + + +@pytest.mark.parametrize( + "cell,field", + [ + ("plain-extra-headers", "extra_headers"), + ("plain-headers", "headers"), + ("plain-provider-specific-header", "provider_specific_header"), + ], +) +def test_a_provider_401_on_the_deployments_own_key_stays_out_when_the_caller_sends_a_plain_header( + rig: Rig, cell: str, field: str +) -> None: + trace: Final = f"{cell}-{uuid.uuid4().hex}" + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name(cell), "messages": messages(cell), field: caller_header(field, "X-Caller-Trace", trace)}, + ) + assert response.status_code == 401, response.text + reached: Final = rig.upstream.drain() + assert [(request.headers["authorization"], request.headers.get("x-caller-trace")) for request in reached] == [ + (f"Bearer sk-deployment-credential-{cell}", trace) + ] + assert_kept_out_of_the_deployment_metrics(rig, cell) + + +def test_a_provider_401_on_a_provider_credential_header_the_proxy_forwards_stays_out_of_the_deployment_metrics( + rig: Rig, +) -> None: + caller_key: Final = f"AIza-forwarded-{uuid.uuid4().hex}" + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name("forwarded-provider-header"), "messages": messages("forwarded-provider-header")}, + headers={"x-goog-api-key": caller_key}, + ) + assert response.status_code == 401, response.text + assert [request.headers.get("x-goog-api-key") for request in rig.upstream.drain()] == [caller_key] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name("forwarded-provider-header")) + observed: Final = recorded(samples, model_id=identity("forwarded-provider-header")) + assert values(samples, FAILURES, model_id=identity("forwarded-provider-header")) == [], observed + assert values(samples, TOTALS, model_id=identity("forwarded-provider-header")) == [], observed + assert values(samples, LLM_FAILED, model_id=identity("forwarded-provider-header")) == [], observed + assert values(samples, STATE, model_id=identity("forwarded-provider-header")) == [], observed + + +def test_a_provider_401_on_the_deployments_own_key_stays_out_when_the_proxy_forwards_a_plain_header(rig: Rig) -> None: + trace: Final = f"forwarded-plain-header-{uuid.uuid4().hex}" + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name("forwarded-plain-header"), "messages": messages("forwarded-plain-header")}, + headers={"x-caller-trace": trace}, + ) + assert response.status_code == 401, response.text + assert [ + (request.headers["authorization"], request.headers.get("x-caller-trace")) for request in rig.upstream.drain() + ] == [("Bearer sk-deployment-credential-forwarded-plain-header", trace)] + assert_kept_out_of_the_deployment_metrics(rig, "forwarded-plain-header") + + +def test_a_provider_401_on_the_deployments_own_key_counts_when_the_proxy_forwards_only_its_own_user_information( + rig: Rig, +) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, "forwarded-user-information") + assert response.status_code == 401, response.text + reached: Final = rig.upstream.drain() + assert [ + (request.headers["authorization"], "x-litellm-user_api_key_hash" in request.headers) for request in reached + ] == [("Bearer sk-deployment-credential-forwarded-user-information", True)] + assert_recorded_as_a_deployment_failure(rig, "forwarded-user-information", "401") + + +def test_a_provider_401_on_a_gateway_key_the_proxy_forwards_stays_out_of_a_healthy_deployments_metrics( + rig: Rig, +) -> None: + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name("forwarded-gateway-key"), "messages": messages("forwarded-gateway-key")}, + headers={"x-portkey-api-key": REVOKED_GATEWAY_KEY}, + ) + assert response.status_code == 401, response.text + assert [ + (request.headers["authorization"], request.headers.get("x-portkey-api-key")) for request in rig.upstream.drain() + ] == [("Bearer sk-deployment-credential-forwarded-gateway-key", REVOKED_GATEWAY_KEY)] + assert_kept_out_of_the_deployment_metrics(rig, "forwarded-gateway-key") + + +@pytest.mark.parametrize( + "cell,body,headers", + [ + ("caller-organization", {"organization": UNKNOWN_ORGANIZATION}, {}), + ("forwarded-organization", {}, {"OpenAI-Organization": UNKNOWN_ORGANIZATION}), + ], +) +def test_a_provider_401_on_an_organization_the_caller_sends_stays_out_of_a_healthy_deployments_metrics( + rig: Rig, cell: str, body: dict[str, JsonValue], headers: dict[str, str] +) -> None: + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", "/v1/chat/completions", {"model": name(cell), "messages": messages(cell), **body}, headers=headers + ) + assert response.status_code == 401, response.text + assert [ + (request.headers["authorization"], request.headers.get("openai-organization")) + for request in rig.upstream.drain() + ] == [(f"Bearer sk-deployment-credential-{cell}", UNKNOWN_ORGANIZATION)] + assert_kept_out_of_the_deployment_metrics(rig, cell) + + +@pytest.mark.parametrize( + ("cell", "headers"), + [ + ("anthropic-own-key", {}), + ("anthropic-sdk-own-key", {"anthropic-version": "2023-06-01", "anthropic-beta": "tools-2024-04-04"}), + ], + ids=["no-client-headers", "anthropic-sdk-headers"], +) +def test_a_provider_401_on_an_anthropic_deployments_own_key_is_recorded( + rig: Rig, cell: str, headers: dict[str, str] +) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(models=[name(cell)]) + rig.upstream.drain() + response: Final = anthropic_messages(rig.gateway, cell, key, headers) + assert response.status_code == 401, response.text + assert [request.headers.get("x-api-key") for request in rig.upstream.drain()] == [ + f"sk-ant-api03-deployment-credential-{cell}" + ] + assert_recorded_as_a_deployment_failure(rig, cell, "401") + + +def test_a_provider_401_on_an_anthropic_oauth_token_the_caller_sent_stays_out_of_the_deployment_metrics( + rig: Rig, +) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(models=[name("caller-oauth")]) + rig.upstream.drain() + response: Final = anthropic_messages( + rig.gateway, + "caller-oauth", + CALLER_OAUTH_TOKEN, + {"x-litellm-api-key": key}, + ) + assert response.status_code == 401, response.text + assert [request.headers.get("authorization") for request in rig.upstream.drain()] == [ + f"Bearer {CALLER_OAUTH_TOKEN}" + ] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name("caller-oauth")) + assert {metric: values(samples, metric, model_id=identity("caller-oauth")) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + }, recorded(samples, requested_model=name("caller-oauth")) + + +def test_a_provider_401_on_an_azure_deployments_own_ad_token_is_recorded(rig: Rig) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, "azure-own-token") + assert response.status_code == 401, response.text + assert [request.headers.get("authorization") for request in rig.upstream.drain()] == [ + "Bearer deployment-entra-token-azure-own-token" + ] + assert_recorded_as_a_deployment_failure(rig, "azure-own-token", "401") + + +def caller_azure_request(cell: str, route: str, credential: dict[str, JsonValue]) -> tuple[str, dict[str, JsonValue]]: + if route == "embeddings": + return "/v1/embeddings", {"model": name(cell), "input": cell, **credential} + if route == "responses": + return "/v1/responses", {"model": name(cell), "input": cell, **credential} + if route == "messages": + return "/v1/messages", {"model": name(cell), "max_tokens": 16, "messages": messages(cell), **credential} + return "/v1/chat/completions", {"model": name(cell), "messages": messages(cell), **credential} + + +@pytest.mark.parametrize( + ("route", "inside_extra_body"), + CALLER_AZURE_CELLS, + ids=[f"{route}{'-extra-body' if inside_extra_body else ''}" for route, inside_extra_body in CALLER_AZURE_CELLS], +) +@pytest.mark.parametrize("opt_in", ("deployment", "proxy-wide")) +def test_a_provider_401_on_an_azure_ad_token_the_caller_sent_stays_out_of_the_deployment_metrics( + request: pytest.FixtureRequest, opt_in: str, route: str, inside_extra_body: bool +) -> None: + opted: Final[Rig] = request.getfixturevalue("rig" if opt_in == "deployment" else "opted_in") + cell: Final = caller_azure_cell(opt_in, route, inside_extra_body) + token: Final = f"caller-entra-token-{uuid.uuid4().hex}" + credential: Final[dict[str, JsonValue]] = ( + {"extra_body": {"azure_ad_token": token}} if inside_extra_body else {"azure_ad_token": token} + ) + opted.upstream.drain() + response: Final = opted.gateway.request("POST", *caller_azure_request(cell, route, credential)) + assert response.status_code == 401, response.text + assert [reached.headers.get("authorization") for reached in opted.upstream.drain()] == [ + None if inside_extra_body else f"Bearer {token}" + ] + samples: Final = until(opted.gateway, PROXY_FAILED, requested_model=name(cell)) + assert {metric: values(samples, metric, model_id=identity(cell)) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + }, recorded(samples, requested_model=name(cell)) + + +@pytest.mark.parametrize(("cell", "field"), CALLER_CREDENTIAL_CELLS, ids=[cell for cell, _ in CALLER_CREDENTIAL_CELLS]) +def test_a_provider_401_on_a_provider_credential_the_caller_sent_stays_out_of_the_deployment_metrics( + rig: Rig, cell: str, field: str +) -> None: + caller_credential: Final = f"caller-credential-{uuid.uuid4().hex}" + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", "/v1/chat/completions", {"model": name(cell), "messages": messages(cell), field: caller_credential} + ) + assert response.status_code == 401, response.text + assert [reached.headers.get("authorization") for reached in rig.upstream.drain()] == [f"Bearer {caller_credential}"] + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name(cell)) + assert {metric: values(samples, metric, model_id=identity(cell)) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + }, recorded(samples, requested_model=name(cell)) + + +@pytest.mark.parametrize("cell", TOKEN_EXCHANGE_CELLS) +def test_a_token_exchange_deployment_rejecting_a_caller_without_an_identity_token_stays_out_of_the_deployment_metrics( + rig: Rig, cell: str +) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, cell) + assert response.status_code == 401, response.text + assert "IdP-issued access token" in response.text, response.text + assert rig.upstream.drain() == () + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name(cell)) + assert {metric: values(samples, metric, model_id=identity(cell)) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + }, recorded(samples, requested_model=name(cell)) + + +@pytest.mark.parametrize("cell", KEYLESS_CELLS) +def test_a_missing_deployment_key_reported_before_the_provider_is_called_stays_out_of_the_deployment_metrics( + rig: Rig, cell: str +) -> None: + rig.upstream.drain() + response: Final = send(rig.gateway, cell) + assert response.status_code == 401, response.text + assert "Missing Anthropic API Key" in response.text, response.text + assert rig.upstream.drain() == () + assert_kept_out_of_the_deployment_metrics(rig, cell) + + +@pytest.mark.parametrize("opt_in", ("deployment", "proxy-wide")) +def test_a_provider_401_at_a_bedrock_endpoint_the_caller_sends_stays_out_of_the_deployment_metrics( + request: pytest.FixtureRequest, opt_in: str +) -> None: + opted: Final[Rig] = request.getfixturevalue("rig" if opt_in == "deployment" else "opted_in") + cell: Final = f"{opt_in}-caller-bedrock-endpoint" + opted.upstream.drain() + response: Final = opted.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name(cell), "messages": messages(cell), "aws_bedrock_runtime_endpoint": opted.upstream.url}, + ) + assert response.status_code == 401, response.text + assert [reached.target for reached in opted.upstream.drain()] == [f"/model/{REJECTS}/converse"] + assert_kept_out_of_the_deployment_metrics(opted, cell) + + +def test_a_provider_401_at_an_api_base_the_caller_sends_counts_against_a_one_off_deployment(rig: Rig) -> None: + caller_api_base: Final = f"{rig.upstream.url}/caller-endpoint-{RUN}/v1" + rig.upstream.drain() + response: Final = rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": name("caller-api-base"), "messages": messages("caller-api-base"), "api_base": caller_api_base}, + ) + assert response.status_code == 401, response.text + assert [(reached.target, reached.headers["authorization"]) for reached in rig.upstream.drain()] == [ + (f"/caller-endpoint-{RUN}/v1/chat/completions", "Bearer sk-deployment-credential-caller-api-base") + ] + samples: Final = until(rig.gateway, FAILURES, api_base=caller_api_base) + one_off: Final = {sample.labels["model_id"] for sample in series(samples, FAILURES, api_base=caller_api_base)} + assert len(one_off) == 1 and identity("caller-api-base") not in one_off, recorded(samples, api_base=caller_api_base) + assert values(samples, FAILURES, api_base=caller_api_base, exception_status="401") == [1.0] + assert values(samples, TOTALS, api_base=caller_api_base) == [1.0] + assert values(samples, STATE, api_base=caller_api_base) == [1.0] + assert values(samples, LLM_FAILED, model_id=next(iter(one_off))) == [1.0] + assert {metric: values(samples, metric, model_id=identity("caller-api-base")) for metric in DEPLOYMENT_SERIES} == { + metric: [] for metric in DEPLOYMENT_SERIES + } + + +def _gated_key(scenario: Scenario, cell: str) -> str | None: + if cell == "invalid-key": + return f"sk-not-a-virtual-key-{uuid.uuid4().hex}" + if cell == "missing-key": + return None + if cell == "no-model-access": + return scenario.key(models=[name("healthy")]) + key: Final = scenario.key(models=[name(cell)]) + if cell == "expired-key": + write_rows( + "UPDATE \"LiteLLM_VerificationToken\" SET expires = now() - interval '1 hour' WHERE token = %s", + (sha256(key.encode()).hexdigest(),), + database_url=scenario.gateway.database_url, + ) + return key + scenario.gateway.post("/key/block", {"key": key}) + return key + + +def _gated(gateway: Gateway, cell: str, key: str | None) -> httpx.Response: + if key is None: + return gateway.client.post("/v1/chat/completions", json={"model": name(cell), "messages": messages(cell)}) + return send(gateway, cell, key=key) + + +def recorded_before(cell: str) -> list[tuple[str, str, str, float]]: + if cell in ("invalid-key", "expired-key"): + return [] + return sorted([(FAILURES, "", "", 1.0), (TOTALS, "", "", 1.0), (LLM_FAILED, identity(cell), "", 1.0)]) + + +def deployment_rows(samples: Sequence[Sample], cell: str) -> list[tuple[str, str, str, float]]: + return sorted( + (sample.name, sample.labels.get("model_id", ""), sample.labels.get("api_base", ""), sample.value) + for sample in samples + if sample.name in DEPLOYMENT_SERIES + and name(cell) in (sample.labels.get("requested_model"), sample.labels.get("model")) + ) + + +@pytest.mark.parametrize("cell", GATED_CELLS) +def test_a_request_the_proxy_rejects_itself_never_marks_the_deployment_down_and_is_recorded_as_before( + rig: Rig, cell: str +) -> None: + with rig.gateway.scenario() as scenario: + key: Final = _gated_key(scenario, cell) + rig.upstream.drain() + response: Final = _gated(rig.gateway, cell, key) + assert response.status_code == (403 if cell == "no-model-access" else 401), response.text + assert rig.upstream.drain() == () + samples: Final = until(rig.gateway, PROXY_FAILED, requested_model=name(cell)) + assert deployment_rows(samples, cell) == recorded_before(cell), recorded(samples, requested_model=name(cell)) + + +def test_a_key_rate_limit_rejected_at_the_proxy_is_still_counted_as_before(rig: Rig) -> None: + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(models=[name("key-rpm-limit")], rpm_limit=1) + rig.upstream.drain() + admitted: Final = send(rig.gateway, "key-rpm-limit", key=key) + assert admitted.status_code == 200, admitted.text + limited: Final = send(rig.gateway, "key-rpm-limit", key=key) + assert limited.status_code == 429, limited.text + assert len(rig.upstream.drain()) == 1 + samples: Final = until(rig.gateway, FAILURES, requested_model=name("key-rpm-limit"), exception_status="429") + assert values(samples, FAILURES, requested_model=name("key-rpm-limit"), exception_status="429") == [1.0] + eventually( + lambda: scrape(rig.gateway), + lambda observed: ( + values(observed, STATE, model_id=identity("key-rpm-limit"), api_base=rig.api_base) == [0.0] + ), + ) + + +@pytest.fixture(scope="module") +def mcp_peer() -> Iterator[McpPeer]: + with scripted_peer(echo_tool("add")) as peer: + yield peer + + +def test_an_mcp_tool_call_the_proxy_rejects_for_the_caller_stays_out_of_the_llm_failure_metrics( + rig: Rig, mcp_peer: McpPeer +) -> None: + alias: Final = "a365" + RUN + with rig.gateway.scenario() as scenario: + server: Final = register_mcp(scenario, mcp_peer, alias) + key: Final = scenario.key(object_permission={"mcp_servers": [server]}) + hashed: Final = sha256(key.encode()).hexdigest() + outcomes: Final = { + entry: McpCaller(rig.gateway, key, entry, alias).call(f"{alias}-add", {"entry": entry}, server) + for entry in ENTRY_POINTS + } + assert all( + outcome.error is not None and "Agent 365 guardrail rejected" in outcome.raw for outcome in outcomes.values() + ), {entry: outcome.raw for entry, outcome in outcomes.items()} + eventually( + lambda: rig.gateway.rows(GUARDRAIL_ROWS, (AGENT_365_GUARDRAIL, hashed, "call_mcp_tool")), + lambda rows: [row["status"] for row in rows] == ["guardrail_intervened"] * len(ENTRY_POINTS), + seconds=70, + ) + samples: Final = scrape(rig.gateway) + assert values(samples, LLM_FAILED, hashed_api_key=hashed) == [] + assert values(samples, FAILURES, hashed_api_key=hashed) == [] + assert values(samples, TOTALS, hashed_api_key=hashed) == [] diff --git a/tests/integration/observability/test_prometheus_provider_auth_failure_chaos.py b/tests/integration/observability/test_prometheus_provider_auth_failure_chaos.py new file mode 100644 index 00000000000..a1fe1cf4f33 --- /dev/null +++ b/tests/integration/observability/test_prometheus_provider_auth_failure_chaos.py @@ -0,0 +1,147 @@ +import threading +import uuid +from collections.abc import Iterator, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import psutil +from integration._support.client import Gateway, eventually +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.prometheus_series import Sample, scrape +from integration._support.wire import Wire, wire_server +from test_prometheus_provider_auth_failure import ( + FAILURES, + LLM_FAILED, + STATE, + TOTALS, + Provider, + config, + identity, + name, + send, + values, +) + +BURST: Final = 12 +ENDPOINTS: Final = ("chat", "chat-stream", "messages", "responses") + + +@dataclass(frozen=True, slots=True) +class ChaosRig: + owned: OwnedProxy + upstream: Wire + release: threading.Event + + +@contextmanager +def _rig(gateway: Gateway, tmp_path: Path) -> Iterator[ChaosRig]: + release: Final = threading.Event() + prom_dir: Final = tmp_path / "prom" + prom_dir.mkdir() + with ( + wire_server(Provider(release)) as upstream, + owned_proxy_process( + gateway, + tmp_path, + {"PROMETHEUS_MULTIPROC_DIR": str(prom_dir)}, + config=config(tmp_path, upstream), + workers=2, + ) as owned, + ): + try: + yield ChaosRig(owned, upstream, release) + finally: + release.set() + + +def _status(gateway: Gateway, cell: str, model: str, key: str | None = None) -> int: + try: + return send(gateway, cell, model=model, key=key).status_code + except httpx.HTTPError: + return -1 + + +def _holds_a_connection_to(pid: int, port: int) -> bool: + return any(connection.raddr[1:] == (port,) for connection in psutil.Process(pid).net_connections(kind="tcp")) + + +def _workers_calling_upstream(rig: ChaosRig) -> frozenset[int]: + port: Final = int(rig.upstream.url.rsplit(":", 1)[1]) + return frozenset(pid for pid in rig.owned.worker_pids() if _holds_a_connection_to(pid, port)) + + +def _counted(samples: Sequence[Sample], cell: str) -> tuple[float, float, float]: + return ( + sum(values(samples, FAILURES, model_id=identity(cell))), + sum(values(samples, TOTALS, model_id=identity(cell))), + sum(values(samples, LLM_FAILED, model_id=identity(cell))), + ) + + +def test_a_burst_of_provider_401s_and_proxy_rejects_across_two_workers_counts_only_the_provider_401s( + gateway: Gateway, tmp_path: Path +) -> None: + with _rig(gateway, tmp_path) as rig: + rig.release.set() + workers: Final = eventually(rig.owned.worker_pids, lambda pids: len(pids) == 2) + invalid: Final = f"sk-not-a-virtual-key-{uuid.uuid4().hex}" + calls: Final[tuple[tuple[str, str, str | None], ...]] = tuple( + (ENDPOINTS[index % len(ENDPOINTS)], name("burst"), None) for index in range(BURST) + ) + tuple(("chat", name("invalid-key"), invalid) for _ in range(BURST)) + + def call(cell: str, model: str, key: str | None) -> int: + return _status(rig.owned.gateway, cell, model, key) + + with ThreadPoolExecutor(max_workers=8) as pool: + statuses: Final = tuple(pool.map(call, *zip(*calls, strict=True))) + assert statuses == (401,) * (2 * BURST), statuses + assert len(rig.upstream.drain()) == BURST + samples: Final = eventually( + lambda: scrape(rig.owned.gateway), + lambda observed: _counted(observed, "burst") == (BURST, BURST, BURST), + seconds=70, + return_last_on_timeout=True, + ) + assert _counted(samples, "burst") == (BURST, BURST, BURST), (workers, _counted(samples, "burst")) + assert set(values(samples, STATE, model_id=identity("burst"))) == {1.0} + assert values(samples, FAILURES, requested_model=name("invalid-key")) == [] + assert values(samples, LLM_FAILED, model=name("invalid-key")) == [] + + +def test_a_worker_killed_while_provider_401s_are_in_flight_loses_no_answered_failure( + gateway: Gateway, tmp_path: Path +) -> None: + with _rig(gateway, tmp_path) as rig: + eventually(rig.owned.worker_pids, lambda pids: len(pids) == 2) + with ThreadPoolExecutor(max_workers=8) as pool: + futures: Final = tuple(pool.submit(_status, rig.owned.gateway, "chat", name("held")) for _ in range(BURST)) + eventually(lambda: rig.upstream.received.qsize() >= 8, bool, seconds=30) + victim: Final = min(eventually(lambda: _workers_calling_upstream(rig), bool)) + with rig.owned.replacing_worker(victim) as kill: + kill() + rig.release.set() + statuses: Final = tuple(future.result() for future in futures) + reached: Final = len(rig.upstream.drain()) + answered: Final = statuses.count(401) + lost: Final = statuses.count(-1) + assert answered + lost == BURST and lost > 0, statuses + samples: Final = eventually( + lambda: scrape(rig.owned.gateway), + lambda observed: answered <= _counted(observed, "held")[0] <= reached, + seconds=70, + return_last_on_timeout=True, + ) + counted: Final = _counted(samples, "held") + assert answered <= counted[0] <= reached and counted[0] == counted[1] == counted[2], (statuses, counted) + follow_up: Final = _status(rig.owned.gateway, "chat", name("held")) + assert follow_up == 401 + recounted: Final = eventually( + lambda: _counted(scrape(rig.owned.gateway), "held"), + lambda observed: observed[0] == counted[0] + 1, + seconds=70, + ) + assert recounted == (counted[0] + 1,) * 3 diff --git a/tests/integration/routing/test_team_wildcard_precedence.py b/tests/integration/routing/test_team_wildcard_precedence.py new file mode 100644 index 00000000000..38fda682129 --- /dev/null +++ b/tests/integration/routing/test_team_wildcard_precedence.py @@ -0,0 +1,61 @@ +"""A team's own wildcard deployment serves that team's keys even when a proxy-wide wildcard of the +same pattern exists, and the proxy-wide one serves every other key (#48490).""" + +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway, Scenario + +pytestmark: Final = pytest.mark.timeout(120) + + +def _wildcard(scenario: Scenario, pattern: str, *, team_id: str | None) -> str: + identity: Final = f"wildcard-{uuid.uuid4().hex}" + scenario.gateway.post( + "/model/new", + { + "model_name": pattern, + "litellm_params": { + "model": "openai/*", + "api_key": f"provider-key-{identity}", + "api_base": f"{scenario.gateway.upstream_url}/v1", + }, + "model_info": {"id": identity, **({"team_id": team_id} if team_id is not None else {})}, + }, + ) + scenario.cleanups.callback(scenario.delete_model, identity) + return identity + + +def _served_by(gateway: Gateway, key: str, model: str) -> str: + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "wildcard precedence control"}]}, + key=key, + ) + assert response.status_code == 200, response.text + return response.headers["x-litellm-model-id"] + + +def test_a_teams_wildcard_serves_its_keys_before_a_proxy_wide_wildcard_of_the_same_pattern(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + byok_team: Final = scenario.team(models=[]) + other_team: Final = scenario.team(models=[]) + pattern: Final = f"byok-{uuid.uuid4().hex}/*" + team_wildcard: Final = _wildcard(scenario, pattern, team_id=byok_team) + shared_wildcard: Final = _wildcard(scenario, pattern, team_id=None) + model: Final = pattern.replace("*", "gpt-4o-mini") + + served: Final = { + "byok team key": _served_by(gateway, scenario.key(team_id=byok_team), model), + "other team key": _served_by(gateway, scenario.key(team_id=other_team), model), + "key without a team": _served_by(gateway, scenario.key(), model), + } + + assert served == { + "byok team key": team_wildcard, + "other team key": shared_wildcard, + "key without a team": shared_wildcard, + } diff --git a/tests/integration/sdk/test_prometheus_copilot_session_auth_failure_sdk.py b/tests/integration/sdk/test_prometheus_copilot_session_auth_failure_sdk.py new file mode 100644 index 00000000000..1b5e990fe41 --- /dev/null +++ b/tests/integration/sdk/test_prometheus_copilot_session_auth_failure_sdk.py @@ -0,0 +1,99 @@ +"""A provider 401 on a caller's own per-user GitHub Copilot session is the caller's credential failing, so the +shared Copilot deployment keeps no failure, request or outage series for it, while a provider 401 on the router +deployment's own key, on a key a router caller passes or on the key a direct SDK call brings is counted as a failed +request, the router caller's key against its own one-off deployment.""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from typing import Final + +import pytest +from integration._support.copilot_session_prometheus_worker import ( + CALLER_SESSION_TOKEN, + COPILOT_DEPLOYMENT, + DEPLOYMENT_SERIES, + DIRECT_CALL, + DIRECT_KEY, + FAILURES, + LLM_FAILED, + OPERATOR_DEPLOYMENT, + OPERATOR_KEY, + ROUTER_CALLER_KEY, + STATE, + TOTALS, + Outcome, + WorkerReport, + run_copilot_session_worker, +) +from integration._support.wire import Reply, Request, wire_server + + +@pytest.fixture(scope="module") +def ran() -> Iterator[tuple[WorkerReport, tuple[Request, ...]]]: + def reject(request: Request) -> Reply: + assert request.method == "POST", request.method + return Reply( + status=401, + body=json.dumps({"error": {"message": "bad credentials", "type": "authentication_error"}}).encode(), + ) + + with wire_server(reject) as upstream: + report: Final = run_copilot_session_worker(upstream.url) + yield report, upstream.drain() + + +def test_every_call_reaches_the_provider_with_its_own_credential_and_fails_with_its_401( + ran: tuple[WorkerReport, tuple[Request, ...]], +) -> None: + report, received = ran + assert report.outcomes == ( + Outcome(deployment=COPILOT_DEPLOYMENT, status=401), + Outcome(deployment=OPERATOR_DEPLOYMENT, status=401), + Outcome(deployment=OPERATOR_DEPLOYMENT, status=401), + Outcome(deployment=DIRECT_CALL, status=401), + ) + assert [r.headers.get("authorization") for r in received] == [ + f"Bearer {CALLER_SESSION_TOKEN}", + f"Bearer {OPERATOR_KEY}", + f"Bearer {ROUTER_CALLER_KEY}", + f"Bearer {DIRECT_KEY}", + ] + + +def test_a_provider_401_on_a_callers_copilot_session_stays_out_of_the_shared_deployment_metrics( + ran: tuple[WorkerReport, tuple[Request, ...]], +) -> None: + report, _ = ran + assert {s.metric for s in report.series if s.model_id == COPILOT_DEPLOYMENT} & DEPLOYMENT_SERIES == set() + + +def test_a_provider_401_on_the_deployments_own_key_counts_as_a_deployment_failure( + ran: tuple[WorkerReport, tuple[Request, ...]], +) -> None: + report, _ = ran + assert report.values(FAILURES, OPERATOR_DEPLOYMENT) == (1.0,) + assert report.values(TOTALS, OPERATOR_DEPLOYMENT) == (1.0,) + assert report.values(LLM_FAILED, OPERATOR_DEPLOYMENT) == (1.0,) + assert report.on_the_endpoint(STATE, OPERATOR_DEPLOYMENT) == (1.0,) + + +def test_a_provider_401_on_the_key_a_direct_sdk_call_brings_is_counted_as_a_failed_request( + ran: tuple[WorkerReport, tuple[Request, ...]], +) -> None: + report, _ = ran + assert report.outside_the_router(FAILURES) == (1.0,) + assert report.outside_the_router(TOTALS) == (1.0,) + assert report.outside_the_router(LLM_FAILED) == (1.0,) + + +def test_a_provider_401_on_a_key_a_router_caller_passes_counts_against_its_own_one_off_deployment( + ran: tuple[WorkerReport, tuple[Request, ...]], +) -> None: + report, _ = ran + assert report.on_a_one_off_deployment(FAILURES) == (1.0,) + assert report.on_a_one_off_deployment(TOTALS) == (1.0,) + assert report.on_a_one_off_deployment(LLM_FAILED) == (1.0,) + assert report.values(FAILURES, OPERATOR_DEPLOYMENT) == (1.0,) + assert report.values(TOTALS, OPERATOR_DEPLOYMENT) == (1.0,) diff --git a/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py b/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py new file mode 100644 index 00000000000..dc94c96da05 --- /dev/null +++ b/tests/integration/sdk/test_router_concurrent_deployment_delete_sdk.py @@ -0,0 +1,474 @@ +from __future__ import annotations + +import asyncio +import json +import linecache +import sys +import threading +import uuid +from collections.abc import AsyncIterable, Awaitable, Callable, Iterable +from concurrent.futures import ThreadPoolExecutor +from types import FrameType +from typing import Final, TypeVar + +import pytest +from integration._support.client import object_value +from integration._support.openai_wire import chat_reply, openai_error, responses_reply +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +import litellm +from litellm import Router + +_DEPLOYMENTS: Final = 4 +_TARGET: Final = _DEPLOYMENTS - 1 +_SYNC_SECONDS: Final = 10 +_RATE_LIMITED_KEY: Final = "key-rate-limited" +_BURST_THREADS: Final = 8 +_BURST_CALLS: Final = 24 +_CHURN_UPSERTS: Final = 120 +_UNREPORTED_MODEL_ID: Final = frozenset({"aresponses-stream", "aanthropic-messages"}) +_ROUTER_FILE: Final = Router.__init__.__code__.co_filename +_INDEX_LOCK_LINES: Final = frozenset( + number + for number, line in enumerate(linecache.getlines(_ROUTER_FILE), start=1) + if "with self._deployment_index_lock" in line +) + +T = TypeVar("T") + + +def _peer(request: Request) -> Reply: + if request.headers["authorization"] == f"Bearer {_RATE_LIMITED_KEY}": + return openai_error(429) + body: Final = object_value(json.loads(request.body)) + stream: Final = body.get("stream") is True + if request.target == "/v1/responses": + return responses_reply(f"resp_{uuid.uuid4().hex}", "gpt-4o-mini", "ok", stream=stream) + return chat_reply(f"chatcmpl-{uuid.uuid4().hex}", "gpt-4o-mini", "ok", stream=stream) + + +def _deployment( + index: int, + api_base: str, + team: bool, + *, + model: int | None = None, + public_name: str | None = None, + timeout: float = 10, +) -> dict[str, JsonValue]: + team_info: Final[dict[str, JsonValue]] = ( + {"team_id": "team-1", "team_public_model_name": public_name or f"public-{index}"} if team else {} + ) + return { + "model_name": f"model-{index if model is None else model}", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": api_base, + "api_key": f"key-{index}", + "timeout": timeout, + }, + "model_info": {"id": f"deployment-{index}", **team_info}, + } + + +def _router(wire: Wire, team: bool = False) -> Router: + return Router(model_list=[_deployment(index, f"{wire.url}/v1", team) for index in range(_DEPLOYMENTS)]) + + +def _read_during_held_write(read: Callable[[], T], write: Callable[[], object]) -> T: + """Run `read` on its own thread while `write` waits where the list has shrunk but the index maps have not caught + up. The write resumes once `read` returns or reaches the router's index lock, so a read that skips the lock meets + the half-updated list and a read that takes the lock waits for the write""" + reindex: Final = Router._update_deployment_indices_after_removal.__code__ + held: Final = threading.Event() + release: Final = threading.Event() + settled: Final = threading.Event() + released_in_time: Final[list[bool]] = [] + outcome: Final[list[T]] = [] + failure: Final[list[BaseException]] = [] + + def hold_at_reindex(frame: FrameType, event: str, _arg: object) -> None: + if event == "call" and frame.f_code is reindex and not held.is_set(): + held.set() + released_in_time.append(release.wait(timeout=_SYNC_SECONDS)) + + def run_write() -> None: + previous: Final = sys.getprofile() + sys.setprofile(hold_at_reindex) + try: + write() + finally: + sys.setprofile(previous) + + def watch_read(frame: FrameType, event: str, _arg: object) -> Callable[..., object] | None: + if frame.f_code.co_filename != _ROUTER_FILE: + return None + if event == "line" and frame.f_lineno in _INDEX_LOCK_LINES: + settled.set() + return watch_read + + def run_read() -> None: + previous: Final = sys.gettrace() + sys.settrace(watch_read) + try: + outcome.append(read()) + except BaseException as error: + failure.append(error) + finally: + sys.settrace(previous) + settled.set() + + writer: Final = threading.Thread(target=run_write) + reader: Final = threading.Thread(target=run_read) + writer.start() + try: + assert held.wait(timeout=_SYNC_SECONDS), "the write never reached re-indexing" + reader.start() + assert settled.wait(timeout=_SYNC_SECONDS), "the read neither returned nor reached the index lock" + finally: + release.set() + if reader.ident is not None: + reader.join() + writer.join() + assert released_in_time == [True], "the write was not held for the whole read" + if failure: + raise failure[0] + return outcome[0] + + +def _read_during_held_delete(router: Router, deployment_id: str, read: Callable[[], T]) -> T: + return _read_during_held_write(read, lambda: router.delete_deployment(deployment_id)) + + +def _keys(wire: Wire) -> tuple[str, ...]: + return tuple(request.headers["authorization"] for request in wire.drain()) + + +def _team_metadata(team: bool) -> dict[str, JsonValue]: + return {"user_api_key_team_id": "team-1"} if team else {} + + +def _completion(router: Router, model: str, team: bool) -> str | None: + response: Final = router.completion( + model=model, messages=[{"role": "user", "content": "hi"}], metadata=_team_metadata(team) + ) + return response._hidden_params["model_id"] + + +def _completion_stream(router: Router, model: str, team: bool) -> str | None: + stream: Final = router.completion( + model=model, messages=[{"role": "user", "content": "hi"}], metadata=_team_metadata(team), stream=True + ) + assert isinstance(stream, Iterable), stream + chunks: Final = list(stream) + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "ok", chunks + return stream._hidden_params["model_id"] + + +def _acompletion(router: Router, model: str, team: bool) -> str | None: + async def call() -> str | None: + response: Final = await router.acompletion( + model=model, messages=[{"role": "user", "content": "hi"}], metadata=_team_metadata(team) + ) + return response._hidden_params["model_id"] + + return asyncio.run(call()) + + +def _acompletion_stream(router: Router, model: str, team: bool) -> str | None: + async def call() -> str | None: + stream: Final = await router.acompletion( + model=model, messages=[{"role": "user", "content": "hi"}], metadata=_team_metadata(team), stream=True + ) + assert isinstance(stream, AsyncIterable), stream + chunks: Final = [chunk async for chunk in stream] + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "ok", chunks + return stream._hidden_params["model_id"] + + return asyncio.run(call()) + + +def _responses(router: Router, model: str, team: bool) -> str | None: + response: Final = router.responses(model=model, input="hi", metadata=_team_metadata(team)) + assert response.output_text == "ok", response + return response._hidden_params["model_id"] + + +def _aresponses(router: Router, model: str, team: bool) -> str | None: + async def call() -> str | None: + response: Final = await router.aresponses(model=model, input="hi", metadata=_team_metadata(team)) + assert response.output_text == "ok", response + return response._hidden_params["model_id"] + + return asyncio.run(call()) + + +def _aresponses_stream(router: Router, model: str, team: bool) -> str | None: + async def call() -> None: + stream: Final = await router.aresponses(model=model, input="hi", metadata=_team_metadata(team), stream=True) + assert isinstance(stream, AsyncIterable), stream + events: Final = [event.type async for event in stream] + assert events[-1] == "response.completed", events + + asyncio.run(call()) + return None + + +def _aanthropic_messages(router: Router, model: str, team: bool) -> str | None: + async def call() -> None: + response: Final = await router.aanthropic_messages( + model=model, messages=[{"role": "user", "content": "hi"}], max_tokens=16, metadata=_team_metadata(team) + ) + assert response["content"][0]["text"] == "ok", response + + asyncio.run(call()) + return None + + +_CALLS: Final[dict[str, Callable[[Router, str, bool], str | None]]] = { + "completion": _completion, + "completion-stream": _completion_stream, + "acompletion": _acompletion, + "acompletion-stream": _acompletion_stream, + "responses": _responses, + "aresponses": _aresponses, + "aresponses-stream": _aresponses_stream, + "aanthropic-messages": _aanthropic_messages, +} + + +@pytest.mark.parametrize("call", list(_CALLS)) +@pytest.mark.parametrize("team", [False, True], ids=["model-name", "team-public-name"]) +def test_call_reaches_its_deployment_while_another_thread_deletes_one(call: str, team: bool) -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire, team) + served_by: Final = _read_during_held_delete( + router, + "deployment-0", + lambda: _CALLS[call](router, f"public-{_TARGET}" if team else f"model-{_TARGET}", team), + ) + keys: Final = _keys(wire) + + assert served_by == (None if call in _UNREPORTED_MODEL_ID else f"deployment-{_TARGET}") + assert keys == (f"Bearer key-{_TARGET}",) + assert router.get_model_ids() == [f"deployment-{index}" for index in range(1, _DEPLOYMENTS)] + + +@pytest.mark.parametrize("deleted", [0, _TARGET - 1], ids=["first", "next-to-target"]) +def test_completion_reaches_its_deployment_whichever_earlier_deployment_is_deleted(deleted: int) -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + served_by: Final = _read_during_held_delete( + router, f"deployment-{deleted}", lambda: _completion(router, f"model-{_TARGET}", team=False) + ) + keys: Final = _keys(wire) + + assert served_by == f"deployment-{_TARGET}" + assert keys == (f"Bearer key-{_TARGET}",) + + +def test_team_completion_reaches_the_remaining_deployment_of_a_shared_public_name_while_its_sibling_is_deleted() -> ( + None +): + with wire_server(_peer) as wire: + api_base: Final = f"{wire.url}/v1" + router: Final = Router( + model_list=[ + _deployment(0, api_base, team=False), + _deployment(1, api_base, team=False), + _deployment(2, api_base, team=True, public_name="public-shared"), + _deployment(3, api_base, team=True, public_name="public-shared"), + ] + ) + served_by: Final = _read_during_held_delete( + router, "deployment-2", lambda: _completion(router, "public-shared", team=True) + ) + keys: Final = _keys(wire) + + assert served_by == "deployment-3" + assert keys == ("Bearer key-3",) + + +def test_completion_for_the_deployment_being_deleted_fails_cleanly_and_sends_nothing() -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + with pytest.raises(litellm.BadRequestError): + _read_during_held_delete( + router, + "deployment-0", + lambda: router.completion(model="model-0", messages=[{"role": "user", "content": "hi"}]), + ) + keys: Final = _keys(wire) + + assert keys == () + assert router.get_model_ids() == [f"deployment-{index}" for index in range(1, _DEPLOYMENTS)] + + +def test_provider_error_during_a_concurrent_delete_comes_from_the_requested_deployment() -> None: + with wire_server(_peer) as wire: + api_base: Final = f"{wire.url}/v1" + rate_limited: Final = _deployment(_TARGET, api_base, team=False) + rate_limited["litellm_params"] = {**object_value(rate_limited["litellm_params"]), "api_key": _RATE_LIMITED_KEY} + router: Final = Router( + model_list=[*(_deployment(index, api_base, team=False) for index in range(_TARGET)), rate_limited], + num_retries=0, + ) + with pytest.raises(litellm.RateLimitError): + _read_during_held_delete( + router, + "deployment-0", + lambda: router.completion(model=f"model-{_TARGET}", messages=[{"role": "user", "content": "hi"}]), + ) + served_by: Final = _completion(router, "model-1", team=False) + keys: Final = _keys(wire) + + assert served_by == "deployment-1" + assert keys == (f"Bearer {_RATE_LIMITED_KEY}", "Bearer key-1") + + +def test_two_threads_deleting_the_same_deployment_remove_it_once() -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + second: Final = _read_during_held_delete( + router, "deployment-0", lambda: router.delete_deployment("deployment-0") + ) + served_by: Final = _completion(router, f"model-{_TARGET}", team=False) + keys: Final = _keys(wire) + + assert second is None + assert served_by == f"deployment-{_TARGET}" + assert keys == (f"Bearer key-{_TARGET}",) + assert router.get_model_ids() == [f"deployment-{index}" for index in range(1, _DEPLOYMENTS)] + + +def test_completion_reaches_its_deployment_while_another_thread_upserts_an_earlier_one() -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + changed: Final = litellm.Deployment(**_deployment(0, f"{wire.url}/v1", team=False, timeout=11)) + served_by: Final = _read_during_held_write( + lambda: _completion(router, f"model-{_TARGET}", team=False), lambda: router.upsert_deployment(changed) + ) + keys: Final = _keys(wire) + + assert served_by == f"deployment-{_TARGET}" + assert keys == (f"Bearer key-{_TARGET}",) + assert sorted(router.get_model_ids()) == [f"deployment-{index}" for index in range(_DEPLOYMENTS)] + + +def test_deployment_added_during_a_concurrent_delete_serves_its_own_requests() -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + added: Final = litellm.Deployment(**_deployment(9, f"{wire.url}/v1", team=False)) + _read_during_held_delete(router, "deployment-0", lambda: router.add_deployment(added)) + served: Final = tuple(_completion(router, f"model-{index}", team=False) for index in (9, _TARGET)) + keys: Final = _keys(wire) + + assert served == ("deployment-9", f"deployment-{_TARGET}") + assert keys == ("Bearer key-9", f"Bearer key-{_TARGET}") + + +def test_model_list_set_during_a_concurrent_delete_serves_its_own_requests() -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + replacement: Final = [_deployment(index, f"{wire.url}/v1", team=False) for index in range(10, 14)] + _read_during_held_delete(router, "deployment-0", lambda: router.set_model_list(replacement)) + served: Final = tuple(_completion(router, f"model-{index}", team=False) for index in range(10, 14)) + keys: Final = _keys(wire) + + assert served == tuple(f"deployment-{index}" for index in range(10, 14)) + assert keys == tuple(f"Bearer key-{index}" for index in range(10, 14)) + assert router.get_model_ids() == [f"deployment-{index}" for index in range(10, 14)] + + +def _burst_router(api_base: str) -> Router: + """Two deployments per model name, so an upsert of one leaves its model group served.""" + return Router( + model_list=[ + _deployment(index, api_base, team=False, model=index % _DEPLOYMENTS) for index in range(2 * _DEPLOYMENTS) + ] + ) + + +def _churn(router: Router, api_base: str, stop: threading.Event) -> int: + """Upsert the deployments in turn with a changed timeout; each upsert moves one to the end of the list.""" + upserts = 0 + while upserts < _CHURN_UPSERTS and not stop.is_set(): + index, round_number = upserts % (2 * _DEPLOYMENTS), upserts // (2 * _DEPLOYMENTS) + router.upsert_deployment( + litellm.Deployment( + **_deployment(index, api_base, team=False, model=index % _DEPLOYMENTS, timeout=11 + round_number % 2) + ) + ) + upserts += 1 + return upserts + + +def _served_keys(wire: Wire) -> dict[str, str]: + """Which key reached the upstream for each request marker.""" + return { + object_value(json.loads(request.body))["messages"][0]["content"]: request.headers["authorization"] + for request in wire.drain() + } + + +def _assert_each_request_used_its_models_deployment(served: list[tuple[str, str | None]], keys: dict[str, str]) -> None: + for marker, model_id in served: + model: Final = int(marker.split(":")[0]) + assert model_id in (f"deployment-{model}", f"deployment-{model + _DEPLOYMENTS}"), (marker, model_id) + assert keys[marker] == f"Bearer key-{str(model_id).removeprefix('deployment-')}", (marker, keys[marker]) + assert len(keys) == len(served) + + +def test_thread_burst_during_upsert_churn_sends_every_request_with_its_deployments_key() -> None: + with wire_server(_peer) as wire: + api_base: Final = f"{wire.url}/v1" + router: Final = _burst_router(api_base) + stop: Final = threading.Event() + + def call(number: int) -> tuple[str, str | None]: + marker: Final = f"{number % _DEPLOYMENTS}:{number}" + response: Final = router.completion( + model=f"model-{number % _DEPLOYMENTS}", messages=[{"role": "user", "content": marker}] + ) + return marker, response._hidden_params["model_id"] + + with ThreadPoolExecutor(max_workers=_BURST_THREADS + 1) as pool: + churn: Final = pool.submit(_churn, router, api_base, stop) + served: Final = list(pool.map(call, range(_BURST_THREADS * _BURST_CALLS))) + stop.set() + upserts: Final = churn.result() + keys: Final = _served_keys(wire) + + assert upserts > 0 + _assert_each_request_used_its_models_deployment(served, keys) + assert sorted(router.get_model_ids()) == sorted(f"deployment-{index}" for index in range(2 * _DEPLOYMENTS)) + + +async def _gathered(calls: Iterable[Awaitable[tuple[str, str | None]]]) -> list[tuple[str, str | None]]: + return list(await asyncio.gather(*calls)) + + +def test_async_burst_during_upsert_churn_sends_every_request_with_its_deployments_key() -> None: + with wire_server(_peer) as wire: + api_base: Final = f"{wire.url}/v1" + router: Final = _burst_router(api_base) + stop: Final = threading.Event() + + async def call(number: int) -> tuple[str, str | None]: + marker: Final = f"{number % _DEPLOYMENTS}:{number}" + response: Final = await router.acompletion( + model=f"model-{number % _DEPLOYMENTS}", messages=[{"role": "user", "content": marker}] + ) + return marker, response._hidden_params["model_id"] + + with ThreadPoolExecutor(max_workers=1) as pool: + churn: Final = pool.submit(_churn, router, api_base, stop) + served: Final = asyncio.run(_gathered(call(number) for number in range(_BURST_THREADS * _BURST_CALLS))) + stop.set() + upserts: Final = churn.result() + keys: Final = _served_keys(wire) + + assert upserts > 0 + _assert_each_request_used_its_models_deployment(served, keys) + assert sorted(router.get_model_ids()) == sorted(f"deployment-{index}" for index in range(2 * _DEPLOYMENTS)) diff --git a/tests/integration/sdk/test_sync_stream_connection_release_wire.py b/tests/integration/sdk/test_sync_stream_connection_release_wire.py index a8b49d285c3..21774df66d0 100644 --- a/tests/integration/sdk/test_sync_stream_connection_release_wire.py +++ b/tests/integration/sdk/test_sync_stream_connection_release_wire.py @@ -212,3 +212,36 @@ def test_sync_stream_that_raises_mid_stream_releases_the_connection() -> None: assert _open_sessions_settling_to(wire, at_most=0) == 0, ( "a stream that raised mid-way left its TCP session open" ) + + +_STUCK: Final = _Shape( + "hosted_vllm", + "hosted_vllm/stuck-model", + _OPENAI_OPENING, + (_openai_delta("the model is stuck on this", None),) * (10 * litellm.REPEATED_STREAMING_CHUNK_LIMIT), +) + + +def _still_generating_peer(request: Request) -> Reply: + return Reply(content_type="text/event-stream", chunks=(_STUCK.opening, *_STUCK.closing), pause_between_chunks=0.01) + + +def test_sync_stream_stuck_repeating_one_chunk_releases_the_connection() -> None: + with wire_server(_still_generating_peer, keep_alive=True) as wire: + stuck: Final = litellm.completion(**_request_kwargs(_STUCK, wire)) + with pytest.raises(Exception, match="repeating the same chunk"): + for _ in stuck: + pass + assert _open_sessions_settling_to(wire, at_most=0) == 0, "a stream stuck on one chunk left its TCP session open" + + +@pytest.mark.asyncio +async def test_async_stream_stuck_repeating_one_chunk_releases_the_connection() -> None: + with wire_server(_still_generating_peer, keep_alive=True) as wire: + stuck: Final = await litellm.acompletion(**_request_kwargs(_STUCK, wire)) + with pytest.raises(Exception, match="repeating the same chunk"): + async for _ in stuck: + pass + assert await _open_sessions_settling_to_while_the_loop_runs(wire, at_most=0) == 0, ( + "a stream stuck on one chunk left its TCP session open" + ) diff --git a/tests/integration/spend/test_team_member_spend.py b/tests/integration/spend/test_team_member_spend.py index 306138e2bb1..c11dc9db4bf 100644 --- a/tests/integration/spend/test_team_member_spend.py +++ b/tests/integration/spend/test_team_member_spend.py @@ -1,3 +1,4 @@ +import asyncio import os import uuid from collections.abc import Callable, Iterator @@ -7,10 +8,14 @@ from typing import Final import httpx import pytest from integration._support.client import Gateway, Scenario, eventually, object_value, string_value -from integration._support.database import read_rows +from integration._support.database import read_rows, write_rows from pydantic import BaseModel, JsonValue from redis import Redis +from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import PrismaClient, ProxyLogging + def _python_isoformat(value: datetime) -> str: return value.isoformat() @@ -444,9 +449,7 @@ def test_member_update_temp_budget_increase_readmits_a_blocked_member( ) assert added.status_code == 200, added.text default_budget_id: Final = _team_default_budget_id(team) if budget_source == "team-default" else None - default_budget_before: Final = ( - _budget_record(default_budget_id) if default_budget_id is not None else () - ) + default_budget_before: Final = _budget_record(default_budget_id) if default_budget_id is not None else () if default_budget_id is not None: assert len(default_budget_before) == 1, repr(default_budget_before) assert default_budget_before[0].max_budget == 0.05, default_budget_before @@ -503,7 +506,7 @@ def test_member_update_temp_budget_increase_readmits_a_blocked_member( else: assert default_budget_id is not None assert state[0].budget_id != default_budget_id, state - assert state[0].max_budget is None, state + assert state[0].max_budget == 0.05, state default_budget_after: Final = _budget_record(default_budget_id) assert default_budget_after == default_budget_before, repr(default_budget_after) assert default_budget_after[0].max_budget == 0.05, default_budget_after @@ -1024,3 +1027,96 @@ def test_bulk_update_restores_team_default_when_last_member_limit_is_cleared( assert first_observations + second_observations == (provider_model, provider_model), repr( (first_observations, second_observations) ) + + +def _budget_reset_at(budget_id: str) -> str: + rows: Final = read_rows( + 'SELECT budget_reset_at::text AS budget_reset_at FROM "LiteLLM_BudgetTable" WHERE budget_id = %s', (budget_id,) + ) + assert len(rows) == 1, rows + return string_value(rows[0]["budget_reset_at"]) + + +async def _run_scheduled_budget_reset() -> None: + client: Final = PrismaClient( + database_url=os.environ["DATABASE_URL"], proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()) + ) + await client.connect() + try: + await ResetBudgetJob( + ProxyLogging(user_api_key_cache=UserApiKeyCache()), client + ).reset_budget_for_litellm_budget_table() + finally: + await client.db.disconnect() + + +def test_temp_budget_increase_on_a_team_default_member_keeps_the_team_reset_schedule( + gateway: Gateway, upstream: httpx.Client +) -> None: + with gateway.scenario() as scenario: + model, provider_model = _priced_model(scenario) + team: Final = scenario.team(models=[model], team_member_budget=5, team_member_budget_duration="30d") + topped_up: Final = scenario.user() + untouched: Final = scenario.user() + for user in (topped_up, untouched): + added = gateway.request( + "POST", "/team/member_add", {"team_id": team, "member": {"user_id": user, "role": "user"}} + ) + assert added.status_code == 200, added.text + default_budget_id: Final = _team_default_budget_id(team) + for user in (topped_up, untouched): + key = scenario.key(team_id=team, user_id=user) + chat = _chat(gateway, model, key, f"spend before top-up {uuid.uuid4().hex}") + assert chat.status_code == 200, chat.text + observed: Final = _observed_models(upstream) + assert observed == (provider_model, provider_model), repr(observed) + for user in (topped_up, untouched): + accrued = eventually( + lambda user=user: _budget_state(team, user), + lambda values: len(values) == 1 and values[0].spend is not None and values[0].spend > 0, + seconds=70, + ) + assert accrued[0].budget_id == default_budget_id, accrued + + expiry: Final = datetime.now(timezone.utc) + timedelta(days=30) + granted: Final = _member_update( + gateway, + { + "team_id": team, + "user_id": topped_up, + "temp_budget_increase": 1, + "temp_budget_expiry": expiry.isoformat(), + }, + ) + assert granted.temp_budget_increase == 1, granted + topped_up_state: Final = _budget_state(team, topped_up) + assert len(topped_up_state) == 1, topped_up_state + topped_up_budget_id: Final = topped_up_state[0].budget_id + assert topped_up_budget_id is not None and topped_up_budget_id != default_budget_id, topped_up_state + assert topped_up_state[0].max_budget == 5, topped_up_state + assert topped_up_state[0].budget_duration == "30d", topped_up_state + assert topped_up_state[0].temp_budget_increase == 1, topped_up_state + assert _budget_reset_at(topped_up_budget_id) == _budget_reset_at(default_budget_id), topped_up_state + assert _budget_state(team, untouched)[0].budget_id == default_budget_id + + write_rows( + "UPDATE \"LiteLLM_BudgetTable\" SET budget_reset_at = NOW() - INTERVAL '1 hour' WHERE budget_id IN (%s, %s)", + (default_budget_id, topped_up_budget_id), + ) + asyncio.run(_run_scheduled_budget_reset()) + for user in (topped_up, untouched): + after_reset = _budget_state(team, user) + assert len(after_reset) == 1 and after_reset[0].spend == 0, (user, after_reset) + assert _budget_reset_at(topped_up_budget_id) == _budget_reset_at(default_budget_id) + + cleared: Final = _member_update( + gateway, + {"team_id": team, "user_id": topped_up, "temp_budget_increase": None, "temp_budget_expiry": None}, + ) + assert cleared.temp_budget_increase is None, cleared + final_state: Final = _budget_state(team, topped_up) + assert len(final_state) == 1, final_state + assert final_state[0].budget_id == topped_up_budget_id, final_state + assert final_state[0].budget_duration == "30d", final_state + assert final_state[0].max_budget == 5, final_state + assert final_state[0].temp_budget_increase is None, final_state diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index e2385a2e215..5b32008b279 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -400,6 +400,68 @@ async def test_systemone_uses_evaluation_pricing(respx_mock: respx.MockRouter, d ) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("base_url", "expected_api_base"), + [ + ("https://onprem.example/v1", "https://onprem.example/v1"), + ("", "https://fallback.example/v1"), + (None, "https://fallback.example/v1"), + (123, "https://fallback.example/v1"), + ], +) +async def test_openai_decisions_honors_base_url_alias( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + base_url: str | int | None, + expected_api_base: str, +) -> None: + monkeypatch.setenv("OPENAI_BASE_URL", "https://fallback.example/v1") + route: Final = respx_mock.post(f"{expected_api_base}/decisions").respond(json=_OPENAI_RESPONSE) + response: Final = await litellm.adecisions( + model="openai/gpt-6-luna", + input="The package arrived broken.", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + api_key="backend-secret-key", + base_url=base_url, + ) + + assert route.called + assert route.calls[0].request.headers["authorization"] == "Bearer backend-secret-key" + assert response._hidden_params["model"] == "openai/gpt-6-luna" + + +@pytest.mark.asyncio +async def test_openai_systemone_honors_base_url_alias(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://onprem.example/v1/decisions").respond(json=_OPENAI_RESPONSE) + response: Final = await litellm.asystemone( + model="openai/gpt-6-luna", + state="The package arrived broken.", + questions=_QUESTIONS, + api_key="backend-secret-key", + base_url="https://onprem.example/v1", + ) + + assert route.called + assert route.calls[0].request.headers["authorization"] == "Bearer backend-secret-key" + assert response._hidden_params["model"] == "openai/gpt-6-luna" + + +@pytest.mark.asyncio +async def test_openai_decisions_api_base_wins_over_base_url_alias(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond(json=_OPENAI_RESPONSE) + await litellm.adecisions( + model="openai/gpt-6-luna", + input="The package arrived broken.", + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + api_key="test-key", + api_base="https://api.openai.com/v1", + base_url="https://onprem.example/v1", + ) + + assert route.called + + @pytest.mark.asyncio async def test_systemone_falls_back_to_plain_pricing_without_decisions_row( respx_mock: respx.MockRouter, diff --git a/tests/unit/experimental_mcp_client/test_mcp_client.py b/tests/unit/experimental_mcp_client/test_mcp_client.py index b4485bcc5f1..60600afab84 100644 --- a/tests/unit/experimental_mcp_client/test_mcp_client.py +++ b/tests/unit/experimental_mcp_client/test_mcp_client.py @@ -1921,6 +1921,62 @@ async def test_list_tools_classifies_upstream_status_for_post_failures(failure_m assert rate_limit is None +@pytest.mark.asyncio +@pytest.mark.parametrize("paginated", (False, True)) +async def test_sse_catalog_limit_reports_the_failure_and_allows_smaller_pages(paginated: bool) -> None: + from mcp.types import Tool + + limit: Final = 1024 + tools: Final = tuple( + Tool(name=f"tool_{index}", description="private-upstream-secret " * 25, input_schema={"type": "object"}) + for index in range(2) + ) + catalog: Final = tuple(tool.model_dump(by_alias=True, exclude_none=True) for tool in tools) + assert len(json.dumps({"tools": catalog}).encode()) > limit + + def respond(request: httpx2.Request) -> httpx2.Response: + if request.method != "POST": + return httpx2.Response(200) + payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content) + assert isinstance(payload, (JSONRPCRequest, JSONRPCNotification)) + if isinstance(payload, JSONRPCNotification): + return httpx2.Response(202) + if payload.method == "initialize": + return httpx2.Response(200, json={ + "jsonrpc": "2.0", "id": payload.id, "result": { + "protocolVersion": LATEST_HANDSHAKE_VERSION, + "capabilities": {"tools": {}}, "serverInfo": {"name": "catalog", "version": "1"}, + }, + }) + assert payload.method == "tools/list" + cursor: Final = (payload.params or {}).get("cursor") + page: Final = catalog[1:] if cursor == "next-page" else catalog[:1] if paginated else catalog + result: Final = { + "tools": page, + **({"nextCursor": "next-page"} if paginated and cursor is None else {}), + } + event: Final = f"data: {json.dumps({'jsonrpc': '2.0', 'id': payload.id, 'result': result})}\n\n" + assert (len(event.encode()) < limit) == paginated + return httpx2.Response(200, headers={"content-type": "text/event-stream"}, content=event) + + def transport_context(url: str, http_client: httpx2.AsyncClient) -> _TransportContext: + return streamable_http_client(url, http_client=http_client, max_sse_event_size=limit) + + client: Final = _MockTransportClient( + respond, server_url="https://upstream.example/mcp", transport_context=transport_context, + ) + if paginated: + assert await client.list_tools(raise_on_error=True) == list(tools) + return + with pytest.raises(MCPError) as caught: + await client.list_tools(raise_on_error=True) + fault: Final = classify_list_exception(caught.value) + assert fault == ServerListFault(tag="upstream_error", sse_event_limit_bytes=limit) + assert list_fault_http_status(fault) == 502 + assert fault.public_message is not None + assert "private-upstream-secret" not in fault.public_message + + @pytest.mark.asyncio async def test_http_status_check_allows_auth_refresh_before_rejecting() -> None: from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ( diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index c7a057cca5c..782442aad22 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -4,6 +4,7 @@ import asyncio import contextvars import threading import time +import uuid from base64 import b64encode from collections.abc import Mapping from dataclasses import replace @@ -14,10 +15,12 @@ from typing import Final import pytest from opentelemetry import trace as trace_api +from opentelemetry.context import Context from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.sdk.trace.id_generator import RandomIdGenerator from opentelemetry.trace import Status, StatusCode import litellm @@ -1159,12 +1162,36 @@ class TestFanOut: ARIZE_TEAM_PARAMS = {"arize_space_id": "space-team", "arize_api_key": "key-team"} +def caller_context(trace_id: int) -> Context: + return trace_api.set_span_in_context( + trace_api.NonRecordingSpan( + trace_api.SpanContext(trace_id=trace_id, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01)) + ) + ) + + def arize_destination(**rates: str) -> OtelDestination: destination = destination_for("arize", {**ARIZE_TEAM_PARAMS, **rates}) assert destination is not None, "the fixture must resolve for the test to mean anything" return destination +#: Trace ids whose SHA-256 reads 0.40 and 0.80 of the way through its first eight bytes, +#: so a rate of 0.5 keeps the first and drops the second. +KEPT_AT_HALF: Final = 4 +DROPPED_AT_HALF: Final = 5 + + +class _FixedTraceId(RandomIdGenerator): + """Every trace the provider starts gets ``trace_id``.""" + + def __init__(self, trace_id: int) -> None: + self._trace_id = trace_id + + def generate_trace_id(self) -> int: + return self._trace_id + + class TestTenantSampling: """A team's ``arize_success_sampling_rate`` and ``arize_error_sampling_rate`` decide how much of its traffic the fan-out delivers to its space, the way they already decide what @@ -1173,14 +1200,17 @@ class TestTenantSampling: NAMES = {"POST /v1/chat/completions", "auth /v1/chat/completions", "chat gpt-4"} @staticmethod - def _fan_out(dest_exporter, draw=None, global_exporter=None): - provider = TracerProvider() + def _fan_out( + dest_exporter: InMemorySpanExporter, + trace_id: int | None = None, + global_exporter: InMemorySpanExporter | None = None, + ) -> TracerProvider: + provider: Final = TracerProvider() if trace_id is None else TracerProvider(id_generator=_FixedTraceId(trace_id)) if global_exporter is not None: provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "arize")) - kwargs = {"processor_factory": lambda _d: SimpleSpanProcessor(dest_exporter)} - if draw is not None: - kwargs["sampling_draw"] = draw - provider.add_span_processor(TenantFanOutSpanProcessor(**kwargs)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) return provider @staticmethod @@ -1224,7 +1254,7 @@ class TestTenantSampling: def test_an_unset_rate_exports_everything(self): dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.99) + provider: Final = self._fan_out(dest_exporter, trace_id=DROPPED_AT_HALF) self._run(provider, (arize_destination(),)) @@ -1300,10 +1330,10 @@ class TestTenantSampling: def test_a_span_that_ends_after_the_root_follows_the_requests_verdict(self): """A post-call database write that starts after the whole request tree has ended - goes where the rest of the tree went, without a second draw.""" + goes where the rest of the tree went, without a second verdict.""" kept_exporter, dropped_exporter = InMemorySpanExporter(), InMemorySpanExporter() - kept = self._fan_out(kept_exporter, draw=lambda: 0.1) - dropped = self._fan_out(dropped_exporter, draw=lambda: 0.9) + kept: Final = self._fan_out(kept_exporter, trace_id=KEPT_AT_HALF) + dropped: Final = self._fan_out(dropped_exporter, trace_id=DROPPED_AT_HALF) destinations = (arize_destination(arize_success_sampling_rate="0.5"),) def late_tree(provider): @@ -1327,14 +1357,263 @@ class TestTenantSampling: } assert dropped_exporter.get_finished_spans() == () + def test_a_span_that_ends_after_a_failed_root_follows_the_failed_requests_verdict(self): + """A post-call database write after a failed request goes where that request went, + though on its own it would answer to the success rate.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + root.set_status(Status(StatusCode.ERROR, "upstream 500")) + root_context: Final = trace_api.set_span_in_context(root) + with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=root_context): + pass + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "postgres INSERT LiteLLM_SpendLogs", + } + + def test_a_failed_call_after_a_dropped_call_of_its_trace_still_answers_to_the_error_rate(self): + """A process that dropped a successful call of the trace keeps a later failed call of + it at an unset error rate, as a process that only served the failed call does.""" + served_both, served_failed = InMemorySpanExporter(), InMemorySpanExporter() + destinations: Final = (arize_destination(arize_success_sampling_rate="0.5"),) + both: Final = self._fan_out(served_both, trace_id=DROPPED_AT_HALF) + + self._run(both, destinations) + self._run(both, destinations, failed=True) + self._run(self._fan_out(served_failed, trace_id=DROPPED_AT_HALF), destinations, failed=True) + + assert [sorted(s.name for s in exporter.get_finished_spans()) for exporter in (served_both, served_failed)] == [ + sorted(self.NAMES) + ] * 2 + + def test_a_successful_call_after_a_kept_failed_call_of_its_trace_answers_to_the_success_rate(self): + """A process that kept a failed call of the trace drops a later successful call of it + at a zero success rate, as a process that only served the successful call does.""" + served_both, served_success = InMemorySpanExporter(), InMemorySpanExporter() + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + both: Final = self._fan_out(served_both, trace_id=KEPT_AT_HALF) + + self._run(both, destinations, failed=True) + self._run(both, destinations) + self._run(self._fan_out(served_success, trace_id=KEPT_AT_HALF), destinations) + + assert len(served_both.get_finished_spans()) == len(self.NAMES), "only the failed call" + assert all(s.name in self.NAMES for s in served_both.get_finished_spans()) + assert served_success.get_finished_spans() == () + + def test_a_late_child_that_fails_after_its_request_was_dropped_is_dropped_with_it(self): + """A post-call write that fails after its successful request was dropped goes where + the request went, rather than reaching the space on its own at the error rate.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + root_context: Final = trace_api.set_span_in_context(root) + with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=root_context) as write: + write.set_status(Status(StatusCode.ERROR, "connection reset")) + + in_fresh_context(run) + + assert dest_exporter.get_finished_spans() == () + + def test_a_failure_deep_in_a_request_marks_the_whole_request(self): + """A database span that fails under the auth span puts the whole request, root + included, on the error rate.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("auth /v1/chat/completions"): + with tracer.start_as_current_span("postgres SELECT LiteLLM_VerificationToken") as lookup: + lookup.set_status(Status(StatusCode.ERROR, "connection reset")) + with tracer.start_as_current_span("chat gpt-4"): + pass + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + *self.NAMES, + "postgres SELECT LiteLLM_VerificationToken", + } + + def test_a_span_two_levels_down_that_outlives_many_other_spans_still_closes_its_request(self): + """A classifier call under the route span that stays open while the worker serves + more requests than it tracks at once still counts against its own request, so the + request is decided once it ends, on its own outcome.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def other_request(): + set_request_destinations(()) + with get_tracer(provider, "litellm").start_as_current_span("POST /v1/chat/completions"): + pass + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + with tracer.start_as_current_span("route gpt-4"): + with tracer.start_as_current_span("chat classifier"): + for _ in range(otel_providers._MAX_OPEN_REQUESTS + 1): + contextvars.Context().run(other_request) + with tracer.start_as_current_span("chat gpt-4"): + pass + root.set_status(Status(StatusCode.ERROR, "upstream 500")) + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "route gpt-4", + "chat classifier", + "chat gpt-4", + } + + def test_a_failure_two_levels_down_reaches_its_request_however_many_spans_are_open(self): + """A classifier call that fails while another request holds more spans open than + the worker tracks requests marks its own request, which is then kept at the error + rate as a whole.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + others: Final = contextvars.Context() + + def open_other_request(): + set_request_destinations(()) + tracer: Final = get_tracer(provider, "litellm") + root: Final = tracer.start_span("POST /v1/chat/completions") + parent: Final = trace_api.set_span_in_context(root) + return ( + root, + *(tracer.start_span("chat flood", context=parent) for _ in range(otel_providers._MAX_OPEN_REQUESTS)), + ) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("route gpt-4"): + with tracer.start_as_current_span("chat classifier") as classifier: + held_open: Final = others.run(open_other_request) + classifier.set_status(Status(StatusCode.ERROR, "classifier timed out")) + return held_open + + held_open: Final = in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "route gpt-4", + "chat classifier", + } + for span in held_open: + span.end() + + def test_the_fan_out_forgets_a_span_once_it_ends_and_a_request_once_it_is_pushed_out(self): + """The request lookup holds only spans still open, and only those of the requests + the fan-out still counts, so neither ended spans nor a flood of open requests grow + it past the open request bound.""" + fan_out: Final = TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(InMemorySpanExporter()) + ) + provider: Final = TracerProvider() + provider.add_span_processor(fan_out) + + def run(): + set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),)) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("chat gpt-4"): + pass + assert (fan_out._roots, dict(fan_out._open)) == ({}, {}) + root: Final = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context()) + tracer.start_span("route gpt-4", context=trace_api.set_span_in_context(root)) + return tuple( + tracer.start_span(f"POST {index}", context=trace_api.Context()) + for index in range(otel_providers._MAX_OPEN_REQUESTS) + ) + + held_open: Final = in_fresh_context(run) + + assert len(fan_out._roots) == otel_providers._MAX_OPEN_REQUESTS + for span in held_open: + span.end() + assert (fan_out._roots, dict(fan_out._open)) == ({}, {}) + + def test_a_span_starting_under_an_ended_span_below_the_root_follows_its_request(self): + """A post-call write started under the route span after its failed request ended + is kept with the request at the error rate, rather than decided on its own at the + success rate.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + with tracer.start_as_current_span("route gpt-4") as route: + route_context: Final = trace_api.set_span_in_context(route) + root.set_status(Status(StatusCode.ERROR, "upstream 500")) + with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=route_context): + pass + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "route gpt-4", + "postgres INSERT LiteLLM_SpendLogs", + } + + def test_concurrent_calls_of_one_trace_each_answer_to_their_own_outcome(self): + """Two calls of the caller's trace in flight at once on one process, one failing, + are decided as two processes serving one each would decide them.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + remote: Final = trace_api.SpanContext( + trace_id=KEPT_AT_HALF, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01) + ) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + caller: Final = trace_api.set_span_in_context(trace_api.NonRecordingSpan(remote)) + with tracer.start_as_current_span("POST failed", context=caller) as failed: + with tracer.start_as_current_span("POST succeeded", context=caller): + pass + failed.set_status(Status(StatusCode.ERROR, "upstream 500")) + + in_fresh_context(run) + + assert [s.name for s in dest_exporter.get_finished_spans()] == ["POST failed"] + def test_a_late_span_whose_verdict_was_forgotten_is_decided_on_its_own_not_stranded(self): """Once enough other requests have been decided to evict a request's verdict, a span of it that starts late (a post-call database write) opens and closes its own count, - so it is decided on a draw of its own as soon as it ends instead of waiting in a tree - nothing would ever close.""" + so it is decided on its own as soon as it ends instead of waiting in a tree nothing + would ever close.""" dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.1) - destinations = (arize_destination(arize_success_sampling_rate="0.5"),) + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),) def run(): set_request_destinations(destinations) @@ -1353,18 +1632,17 @@ class TestTenantSampling: assert names[0] == "POST /v1/chat/completions" assert names[-1] == "postgres INSERT LiteLLM_SpendLogs", "exported as it ended, not held for a root" - def test_a_callers_traceparent_neither_steers_the_draw_nor_hides_the_root(self): - """The draw is random, not read from the trace id a caller may have chosen, and the - server span under a remote parent still closes the tree.""" + def test_a_callers_traceparent_decides_by_its_trace_id_and_still_closes_the_tree(self): + """A request continuing a caller's trace is decided on that trace id, and the server + span under a remote parent still closes the tree.""" dest_exporter = InMemorySpanExporter() - draws = iter([0.9, 0.1]) - provider = self._fan_out(dest_exporter, draw=lambda: next(draws)) + provider: Final = self._fan_out(dest_exporter) destinations = (arize_destination(arize_success_sampling_rate="0.5"),) def run(): set_request_destinations(destinations) tracer = get_tracer(provider, "litellm") - for trace_id in (1, 2): + for trace_id in (DROPPED_AT_HALF, KEPT_AT_HALF): remote = trace_api.SpanContext( trace_id=trace_id, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01) ) @@ -1377,12 +1655,12 @@ class TestTenantSampling: kept = dest_exporter.get_finished_spans() assert [s.name for s in kept] == ["chat gpt-4", "POST /v1/chat/completions"] - assert all(s.context.trace_id == 2 for s in kept), "the second request, whose draw of 0.1 passed" + assert all(s.context.trace_id == KEPT_AT_HALF for s in kept) def test_a_root_that_never_ends_holds_the_tree_only_up_to_the_bound(self): dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.0) - destinations = (arize_destination(arize_success_sampling_rate="0.5"),) + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),) def run(): set_request_destinations(destinations) @@ -1397,8 +1675,8 @@ class TestTenantSampling: def test_a_flood_of_waiting_trees_decides_the_oldest_on_what_it_has(self): dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.0) - destinations = (arize_destination(arize_success_sampling_rate="0.5"),) + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),) def run(): set_request_destinations(destinations) @@ -1413,21 +1691,21 @@ class TestTenantSampling: assert [s.name for s in dest_exporter.get_finished_spans()] == ["chat 0"] - def test_a_flood_of_open_traces_forgets_the_oldest_and_decides_it_as_its_spans_end(self): - """The fan-out counts open spans for a bounded number of traces, a bound far above - what one process keeps in flight, since every trace of the provider is counted, not - only the sampled ones. A trace pushed out of that count is decided on what it has as + def test_a_flood_of_open_requests_forgets_the_oldest_and_decides_it_as_its_spans_end(self): + """The fan-out counts open spans for a bounded number of requests, a bound far above + what one process keeps in flight, since every request of the provider is counted, not + only the sampled ones. A request pushed out of that count is decided on what it has as its next span ends, root still open, rather than held until a bound gets to it.""" dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.0) - destinations = (arize_destination(arize_success_sampling_rate="0.5"),) + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),) def run(): set_request_destinations(destinations) tracer = get_tracer(provider, "litellm") root = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context()) tracer.start_span("auth /v1/chat/completions", context=trace_api.set_span_in_context(root)).end() - for index in range(otel_providers._MAX_OPEN_TRACES): + for index in range(otel_providers._MAX_OPEN_REQUESTS): tracer.start_span(f"POST {index}", context=trace_api.Context()) assert dest_exporter.get_finished_spans() == (), "still held before the count is forgotten" tracer.start_span("chat gpt-4", context=trace_api.set_span_in_context(root)).end() @@ -1443,14 +1721,12 @@ class TestTenantSampling: def test_shutdown_decides_what_is_still_held(self): dest_exporter = InMemorySpanExporter() - fan_out = TenantFanOutSpanProcessor( - processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter), sampling_draw=lambda: 0.0 - ) + fan_out: Final = TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) provider = TracerProvider() provider.add_span_processor(fan_out) def run(): - set_request_destinations((arize_destination(arize_success_sampling_rate="0.5"),)) + set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),)) tracer = get_tracer(provider, "litellm") with tracer.start_as_current_span("POST /v1/chat/completions"): with tracer.start_as_current_span("chat gpt-4"): @@ -1476,14 +1752,12 @@ class TestTenantSampling: straggler.end() super().on_end(span) - fan_out = TenantFanOutSpanProcessor( - processor_factory=lambda _d: _EndsAStragglerOnExport(dest_exporter), sampling_draw=lambda: 0.0 - ) + fan_out: Final = TenantFanOutSpanProcessor(processor_factory=lambda _d: _EndsAStragglerOnExport(dest_exporter)) provider = TracerProvider() provider.add_span_processor(fan_out) def run(): - set_request_destinations((arize_destination(arize_success_sampling_rate="0.5"),)) + set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),)) tracer = get_tracer(provider, "litellm") root = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context()) tracer.start_span("chat gpt-4", context=trace_api.set_span_in_context(root)).end() @@ -1495,20 +1769,20 @@ class TestTenantSampling: assert {s.name for s in dest_exporter.get_finished_spans()} == {"chat gpt-4", "embed ada"} - def test_a_zero_rate_drops_even_a_draw_of_zero(self): + def test_a_zero_rate_drops_a_trace_a_half_rate_keeps(self): dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.0) + provider: Final = self._fan_out(dest_exporter, trace_id=KEPT_AT_HALF) self._run(provider, (arize_destination(arize_success_sampling_rate="0.0"),)) assert dest_exporter.get_finished_spans() == () - def test_the_draw_is_compared_to_the_rate_the_way_the_legacy_callback_compares_it(self): + def test_the_trace_id_is_read_against_the_rate(self): dropped_exporter, kept_exporter = InMemorySpanExporter(), InMemorySpanExporter() - destinations = (arize_destination(arize_success_sampling_rate="0.2"),) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.5"),) - self._run(self._fan_out(dropped_exporter, draw=lambda: 0.3), destinations) - self._run(self._fan_out(kept_exporter, draw=lambda: 0.2), destinations) + self._run(self._fan_out(dropped_exporter, trace_id=DROPPED_AT_HALF), destinations) + self._run(self._fan_out(kept_exporter, trace_id=KEPT_AT_HALF), destinations) assert dropped_exporter.get_finished_spans() == () assert {s.name for s in kept_exporter.get_finished_spans()} == self.NAMES @@ -1527,6 +1801,39 @@ class TestTenantSampling: assert all(names == self.NAMES for names in by_trace.values()) assert 0 < len(by_trace) < 64 + @pytest.mark.parametrize( + ("rates", "failed"), + [({"arize_success_sampling_rate": "0.5"}, False), ({"arize_error_sampling_rate": "0.5"}, True)], + ids=["success-rate", "error-rate"], + ) + def test_two_processes_serving_one_trace_reach_the_same_verdict(self, rates: Mapping[str, str], failed: bool): + """Two pods behind a load balancer each serve one call of a trace the caller + continued with its ``traceparent``: the team's space gets both calls or neither, + and a rate of 0.5 over UUID-shaped trace ids keeps some traces and drops others.""" + exporters: Final = (InMemorySpanExporter(), InMemorySpanExporter()) + pods: Final = tuple(self._fan_out(exporter) for exporter in exporters) + destinations: Final = (arize_destination(**rates),) + trace_ids: Final = tuple(uuid.uuid5(uuid.NAMESPACE_URL, f"pod/{index}").int for index in range(32)) + + def run(provider: TracerProvider): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + for trace_id in trace_ids: + with tracer.start_as_current_span("POST /v1/chat/completions", context=caller_context(trace_id)): + with tracer.start_as_current_span("chat gpt-4") as span: + if failed: + span.set_status(Status(StatusCode.ERROR, "upstream 500")) + + for provider in pods: + in_fresh_context(run, provider) + + first, second = ( + {s.context.trace_id for s in exporter.get_finished_spans() if s.context is not None} + for exporter in exporters + ) + assert first == second + assert 0 < len(first) < len(trace_ids) + def test_in_additive_mode_the_operators_own_copy_is_not_sampled(self, monkeypatch): """The rates are the team's setting for the team's space; the operator's backbone keeps every span.""" @@ -1547,7 +1854,7 @@ class TestTenantSampling: assert {s.name for s in dest_exporter.get_finished_spans()} == self.NAMES - def test_two_views_of_one_exporter_each_draw_at_their_own_rate(self): + def test_two_views_of_one_exporter_each_answer_to_their_own_rate(self): """Two destinations to the same space with different rates share an exporter, not a verdict: the 1.0 view gets the tree once and the 0.0 view adds nothing.""" dest_exporter = InMemorySpanExporter() @@ -1582,7 +1889,7 @@ class TestTenantSampling: @pytest.mark.parametrize("bad", ["abc", "-0.1", "1.5", "nan"]) def test_an_unusable_rate_exports_rather_than_dropping(self, bad): dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter, draw=lambda: 0.99) + provider: Final = self._fan_out(dest_exporter, trace_id=DROPPED_AT_HALF) self._run(provider, (arize_destination(arize_success_sampling_rate=bad),)) diff --git a/tests/unit/integrations/test_prometheus_invalid_key_filtering.py b/tests/unit/integrations/test_prometheus_invalid_key_filtering.py index 9e8348e860a..453fd28636d 100644 --- a/tests/unit/integrations/test_prometheus_invalid_key_filtering.py +++ b/tests/unit/integrations/test_prometheus_invalid_key_filtering.py @@ -2,19 +2,63 @@ Unit tests for Prometheus invalid API key request filtering. Tests the 401 detection helpers, that LLM-level metrics skip invalid API key -requests, and that the proxy-level failed request counter still records them. +requests rejected at the proxy gate while recording a provider's 401, and that +the proxy-level failed request counter still records them. """ +from collections.abc import Iterator +from datetime import datetime +from typing import Final from unittest.mock import Mock, patch import pytest from fastapi import HTTPException from prometheus_client import REGISTRY +from prometheus_client.samples import Sample +import litellm +from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL from litellm.integrations.prometheus import PrometheusLogger +from litellm.llms.github_copilot.per_user_auth import GithubCopilotUserSession from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth +PROVIDER_CREDENTIAL_FIELDS = [ + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_web_identity_token", + "azure_ad_token", + "azure_password", + "azure_scope", + "azure_username", + "client_id", + "client_secret", + "gigachat_access_token", + "oci_fingerprint", + "oci_key", + "oci_key_file", + "oci_tenancy", + "oci_user", + "tenant_id", + "token", + "user_config", + "vertex_ai_credentials", + "vertex_credentials", + "watsonx_credentials", + "wx_credentials", + "zen_api_key", +] +PROVIDER_ENDPOINT_FIELDS = [ + "aws_bedrock_runtime_endpoint", + "aws_sts_endpoint", + "deployment_url", + "s3_endpoint_url", + "sagemaker_base_url", +] +CALLED_THE_PROVIDER: Final = {"api_call_start_time": datetime(2026, 10, 19, 12, 0, 0)} + + @pytest.fixture(scope="function") def prometheus_logger(): """Create a PrometheusLogger instance for testing.""" @@ -48,18 +92,13 @@ class TestExtractStatusCode: (ExceptionWithStatusCode, 401, 401), ], ) - def test_extract_from_exception( - self, prometheus_logger, exception_class, code_value, expected - ): + def test_extract_from_exception(self, prometheus_logger, exception_class, code_value, expected): exception = exception_class(code_value) assert prometheus_logger._extract_status_code(exception=exception) == expected def test_extract_from_kwargs(self, prometheus_logger): exception = ExceptionWithCode("401") - assert ( - prometheus_logger._extract_status_code(kwargs={"exception": exception}) - == 401 - ) + assert prometheus_logger._extract_status_code(kwargs={"exception": exception}) == 401 def test_extract_from_enum_values(self, prometheus_logger): enum_values = Mock(status_code="401") @@ -79,50 +118,271 @@ class TestInvalidAPIKeyDetection: ], ) def test_status_code_detection(self, prometheus_logger, status_code, expected): - assert ( - prometheus_logger._is_invalid_api_key_request(status_code=status_code) - == expected - ) + assert prometheus_logger._is_invalid_api_key_request(status_code=status_code) == expected def test_auth_error_message_detection(self, prometheus_logger): exception = AssertionError( "LiteLLM Virtual Key expected. Received=invalid-key-12345, expected to start with 'sk-'." ) - assert ( - prometheus_logger._is_invalid_api_key_request( - status_code=None, exception=exception - ) - is True - ) + assert prometheus_logger._is_invalid_api_key_request(status_code=None, exception=exception) is True def test_non_auth_exception_not_detected(self, prometheus_logger): exception = ValueError("Some other error") - assert ( - prometheus_logger._is_invalid_api_key_request( - status_code=None, exception=exception - ) - is False - ) + assert prometheus_logger._is_invalid_api_key_request(status_code=None, exception=exception) is False class TestSkipMetricsValidation: """Test high-level validation method that orchestrates detection and extraction.""" - def test_skip_for_401_exception(self, prometheus_logger): - """Test full flow: extraction -> detection -> skip decision.""" - exception = ExceptionWithCode("401") - assert ( - prometheus_logger._should_skip_metrics_for_invalid_key(exception=exception) - is True - ) + @pytest.mark.parametrize( + "exception,proxy_gate,expected", + [ + (ExceptionWithCode("401"), True, True), + (AssertionError("expected to start with 'sk-'"), True, True), + (ExceptionWithStatusCode(429), True, False), + (ExceptionWithCode("401"), False, True), + (AssertionError("expected to start with 'sk-'"), False, True), + ( + litellm.AuthenticationError( + message="Incorrect API key provided: sk-rotat*****", + llm_provider="openai", + model="gpt-4o-mini", + ), + False, + False, + ), + ( + litellm.AuthenticationError( + message="API key not valid. Please pass a valid API key.", + llm_provider="gemini", + model="gemini-2.5-flash", + ), + False, + False, + ), + ( + litellm.AuthenticationError( + message="Authentication Error, Invalid proxy server token passed.", + llm_provider="openai", + model="gpt-4o-mini", + ), + True, + True, + ), + ( + litellm.CallerCredentialAuthenticationError( + message="GitHub Copilot connection is missing for this user", + llm_provider="github_copilot", + model="gpt-4o", + ), + False, + True, + ), + ( + HTTPException(status_code=401, detail="caller did not present a token"), + False, + True, + ), + ( + litellm.BadRequestError( + message="API key not valid. Please pass a valid API key.", + llm_provider="gemini", + model="gemini-2.5-flash", + ), + False, + False, + ), + ], + ids=[ + "proxy-gate-401", + "proxy-gate-auth-message", + "proxy-gate-429", + "unmarked-proxy-401", + "unmarked-auth-message", + "provider-401", + "provider-auth-message", + "proxy-gate-litellm-auth-error", + "caller-credential-401", + "unmarked-http-exception-401", + "provider-400-auth-message", + ], + ) + def test_skip_except_for_provider_errors_the_proxy_did_not_raise( + self, prometheus_logger, exception, proxy_gate, expected + ): + kwargs: Final[dict[str, object]] = {"exception": exception, **CALLED_THE_PROVIDER} + if proxy_gate: + kwargs[LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL] = True + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected - def test_skip_for_auth_error_message(self, prometheus_logger): - """Test full flow: exception message -> detection -> skip decision.""" - exception = AssertionError("expected to start with 'sk-'") - assert ( - prometheus_logger._should_skip_metrics_for_invalid_key(exception=exception) - is True + @pytest.mark.parametrize( + "logged_call,expected", + [ + ({"litellm_params": {"proxy_server_request": {"caller_credential_fields": ("api_key",)}}}, True), + ( + { + "litellm_params": { + "github_copilot_user_session": GithubCopilotUserSession( + token="caller-session-token", api_base="https://api.githubcopilot.com" + ) + } + }, + True, + ), + ({"standard_logging_object": {"metadata": {"used_client_oauth_token": True}}}, True), + ({"litellm_params": {"proxy_server_request": {"caller_credential_fields": ("headers",)}}}, True), + ( + { + "litellm_params": { + "proxy_server_request": {"caller_credential_fields": ("provider_specific_header",)} + } + }, + True, + ), + ({"litellm_params": {"proxy_server_request": {"caller_credential_fields": ()}}}, False), + ({"litellm_params": {"proxy_server_request": {"caller_credential_fields": "api_key"}}}, False), + ({"standard_logging_object": {"metadata": {"used_client_oauth_token": False}}}, False), + ({"litellm_params": {"proxy_server_request": {"body": {"azure_ad_token": "caller-token"}}}}, True), + ( + { + "litellm_params": { + "proxy_server_request": {"body": {"extra_body": {"azure_ad_token": "caller-token"}}} + } + }, + True, + ), + ( + { + "litellm_params": { + "azure_ad_token": "deployment-token", + "proxy_server_request": {"body": {"model": "azure-deployment"}}, + } + }, + False, + ), + ({"litellm_params": {"proxy_server_request": {"body": {"extra_body": "azure_ad_token"}}}}, False), + ( + { + "litellm_params": { + "proxy_server_request": {"body": {"litellm_credential_name": "caller-credential"}} + } + }, + True, + ), + ( + { + "litellm_params": { + "litellm_credential_name": "deployment-credential", + "proxy_server_request": {"body": {"model": "openai-deployment"}}, + } + }, + False, + ), + ({"litellm_params": {"proxy_server_request": {"caller_credential_fields": ("extra_headers",)}}}, True), + ({"litellm_params": {"proxy_server_request": {"caller_credential_fields": ("organization",)}}}, True), + ( + { + "litellm_params": { + "proxy_server_request": {"credential_fields": ("headers",), "caller_credential_fields": ()} + } + }, + False, + ), + ( + { + "litellm_params": { + "proxy_server_request": {"body": {"extra_headers": {"Authorization": "Bearer sk-caller"}}} + } + }, + False, + ), + ], + ids=[ + "caller-supplied-api-key", + "caller-copilot-session", + "caller-anthropic-oauth-token", + "caller-supplied-headers", + "caller-supplied-provider-specific-header", + "no-caller-credential-fields", + "malformed-credential-fields", + "deployment-anthropic-key", + "caller-supplied-azure-ad-token", + "caller-supplied-azure-ad-token-in-extra-body", + "deployment-azure-ad-token", + "malformed-extra-body", + "caller-named-credential", + "deployment-named-credential", + "caller-supplied-extra-headers", + "caller-supplied-organization", + "headers-that-cannot-change-provider-auth", + "extra-headers-not-stamped-by-the-proxy", + ], + ) + def test_skip_a_provider_401_on_a_credential_the_caller_owns(self, prometheus_logger, logged_call, expected): + exception: Final = litellm.AuthenticationError( + message="Incorrect API key provided: sk-calle*****", + llm_provider="openai", + model="gpt-4o-mini", ) + kwargs: Final = {"exception": exception, **CALLED_THE_PROVIDER, **logged_call} + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected + + @pytest.mark.parametrize( + "logged_call,expected", + [(CALLED_THE_PROVIDER, False), ({}, True), ({"api_call_start_time": None}, True)], + ids=["provider-401", "401-before-the-provider-call", "blank-provider-call-start"], + ) + def test_skip_a_401_raised_before_the_provider_was_called(self, prometheus_logger, logged_call, expected): + exception: Final = litellm.AuthenticationError( + message="Missing Anthropic API Key - A call is being made to anthropic but no key is set", + llm_provider="anthropic", + model="claude-haiku-4-5", + ) + kwargs: Final = {"exception": exception, **logged_call} + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected + + @pytest.mark.parametrize("field", PROVIDER_CREDENTIAL_FIELDS + PROVIDER_ENDPOINT_FIELDS) + @pytest.mark.parametrize("inside_extra_body", [False, True], ids=["body", "extra-body"]) + def test_skip_a_provider_401_on_a_provider_credential_field_the_caller_sent( + self, prometheus_logger, field, inside_extra_body + ): + sent: Final = {field: "caller-credential"} + body: Final = {"model": "shared-deployment", **({"extra_body": sent} if inside_extra_body else sent)} + kwargs: Final = { + "exception": litellm.AuthenticationError(message="Unauthorized", llm_provider="azure", model="gpt-4o"), + "litellm_params": {"proxy_server_request": {"body": body}}, + **CALLED_THE_PROVIDER, + } + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is True + + @pytest.mark.parametrize("field", PROVIDER_CREDENTIAL_FIELDS + PROVIDER_ENDPOINT_FIELDS) + def test_count_a_provider_401_on_a_provider_credential_field_the_deployment_owns(self, prometheus_logger, field): + kwargs: Final = { + "exception": litellm.AuthenticationError(message="Unauthorized", llm_provider="azure", model="gpt-4o"), + **CALLED_THE_PROVIDER, + "litellm_params": { + field: "deployment-credential", + "proxy_server_request": {"body": {"model": "shared-deployment"}}, + }, + } + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is False + + @pytest.mark.parametrize( + "token_exchange_endpoint,expected", + [("https://login.example.com/oauth2/v2.0/token", True), ("", False), (None, False)], + ids=["token-exchange-deployment", "blank-endpoint", "no-endpoint"], + ) + def test_skip_a_provider_401_on_a_token_exchange_deployment( + self, prometheus_logger, token_exchange_endpoint, expected + ): + kwargs: Final = { + "exception": litellm.AuthenticationError( + message="Unauthorized", llm_provider="microsoft_365_copilot", model="copilot" + ), + "litellm_params": {"token_exchange_endpoint": token_exchange_endpoint, "client_secret": "deployment"}, + **CALLED_THE_PROVIDER, + } + assert prometheus_logger._should_skip_metrics_for_invalid_key(kwargs=kwargs) is expected def test_no_skip_for_valid_request(self, prometheus_logger): assert prometheus_logger._should_skip_metrics_for_invalid_key() is False @@ -147,19 +407,13 @@ class TestAsyncHooks: ), ], ) - async def test_post_call_failure_hook_counts_401_without_key_hash( - self, prometheus_logger, exception - ): + async def test_post_call_failure_hook_counts_401_without_key_hash(self, prometheus_logger, exception): unauthenticated = UserAPIKeyAuth(request_route="/v1/chat/completions") unauthenticated.api_key = "notakeyatall" with ( - patch.object( - prometheus_logger, "litellm_proxy_failed_requests_metric" - ) as mock_failed, - patch.object( - prometheus_logger, "litellm_proxy_total_requests_metric" - ) as mock_total, + patch.object(prometheus_logger, "litellm_proxy_failed_requests_metric") as mock_failed, + patch.object(prometheus_logger, "litellm_proxy_total_requests_metric") as mock_total, ): await prometheus_logger.async_post_call_failure_hook( request_data={"model": "test-model"}, @@ -176,9 +430,7 @@ class TestAsyncHooks: mock_total.labels.return_value.inc.assert_called_once() @pytest.mark.asyncio - async def test_post_call_failure_hook_keeps_resolved_identity_labels_for_401( - self, prometheus_logger - ): + async def test_post_call_failure_hook_keeps_resolved_identity_labels_for_401(self, prometheus_logger): expired_key = UserAPIKeyAuth( api_key="sk-expired", key_alias="expired-alias", @@ -191,9 +443,7 @@ class TestAsyncHooks: code=401, ) - with patch.object( - prometheus_logger, "litellm_proxy_failed_requests_metric" - ) as mock_failed: + with patch.object(prometheus_logger, "litellm_proxy_failed_requests_metric") as mock_failed: await prometheus_logger.async_post_call_failure_hook( request_data={"model": "test-model"}, original_exception=exception, @@ -207,7 +457,7 @@ class TestAsyncHooks: assert failed_labels["team"] == "team-1" @pytest.mark.asyncio - async def test_log_failure_event_skips_401(self, prometheus_logger): + async def test_log_failure_event_skips_401_rejected_at_the_proxy_gate(self, prometheus_logger): exception = ExceptionWithCode("401") kwargs = { "model": "test-model", @@ -220,20 +470,91 @@ class TestAsyncHooks: }, "exception": exception, "litellm_params": {}, + LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True, } with ( - patch.object( - prometheus_logger, "litellm_llm_api_failed_requests_metric" - ) as mock_failed, - patch.object( - prometheus_logger, "set_llm_deployment_failure_metrics" - ) as mock_deployment, + patch.object(prometheus_logger, "litellm_llm_api_failed_requests_metric") as mock_failed, + patch.object(prometheus_logger, "set_llm_deployment_failure_metrics") as mock_deployment, ): - await prometheus_logger.async_log_failure_event( kwargs=kwargs, response_obj=None, start_time=None, end_time=None ) mock_failed.labels.assert_not_called() mock_deployment.assert_not_called() + + +def _provider_auth_failure_kwargs() -> dict[str, object]: + exception: Final = litellm.AuthenticationError( + message="Incorrect API key provided: sk-rotat*****", + llm_provider="openai", + model="gpt-4o-mini", + ) + return { + "model": "gpt-4o-mini", + "standard_logging_object": { + "metadata": { + "user_api_key_hash": "test-key", + "user_api_key_alias": "test-alias", + "user_api_key_user_id": "test-user", + "user_api_key_team_id": "test-team", + "user_api_key_team_alias": "test-team-alias", + }, + "model_group": "rotated-credential", + "model_id": "deployment-1", + "api_base": "https://api.openai.com/v1", + "custom_llm_provider": "openai", + }, + "exception": exception, + "litellm_params": {"custom_llm_provider": "openai", "metadata": {}}, + **CALLED_THE_PROVIDER, + } + + +def _metric_values(name: str, model_id: str) -> list[float]: + return [ + sample.value + for sample in _registry_samples() + if sample.name == name and sample.labels.get("model_id") == model_id + ] + + +def _registry_samples() -> Iterator[Sample]: + for metric in REGISTRY.collect(): + yield from metric.samples + + +class TestProviderAuthFailures: + """A 401 raised by the provider is an upstream failure and is recorded.""" + + @pytest.mark.asyncio + async def test_log_failure_event_records_a_provider_401(self, prometheus_logger): + await prometheus_logger.async_log_failure_event( + kwargs=_provider_auth_failure_kwargs(), + response_obj=None, + start_time=None, + end_time=None, + ) + + assert _metric_values("litellm_llm_api_failed_requests_metric_total", "deployment-1") == [1.0] + assert _metric_values("litellm_deployment_failure_responses_total", "deployment-1") == [1.0] + assert _metric_values("litellm_deployment_state", "deployment-1") == [1.0] + + def test_deployment_failure_metrics_record_a_provider_401(self, prometheus_logger): + prometheus_logger.set_llm_deployment_failure_metrics(_provider_auth_failure_kwargs()) + + failures: Final = [ + sample for sample in _registry_samples() if sample.name == "litellm_deployment_failure_responses_total" + ] + assert [ + ( + sample.labels["model_id"], + sample.labels["exception_status"], + sample.labels["api_provider"], + sample.value, + ) + for sample in failures + ] == [("deployment-1", "401", "openai", 1.0)] + assert _metric_values("litellm_deployment_total_requests_total", "deployment-1") == [1.0] + assert _metric_values("litellm_deployment_state", "deployment-1") == [1.0] diff --git a/tests/unit/litellm_core_utils/test_fallback_generalizations.py b/tests/unit/litellm_core_utils/test_fallback_generalizations.py index 7807c438503..e59252e1571 100644 --- a/tests/unit/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/unit/litellm_core_utils/test_fallback_generalizations.py @@ -369,7 +369,7 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp "output_cost_per_token": 2e-6, "litellm_provider": "openai", "mode": "chat", - "max_tokens": 7, + "max_output_tokens": 7, "supports_reasoning": False, }, "acme-bare": { @@ -398,7 +398,7 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp "name": "acme-backfill", "pattern": r"^acme-", "fill_missing_for_providers": ["openai"], - "model_info": {"supports_reasoning": True, "max_tokens": 5}, + "model_info": {"supports_reasoning": True, "max_output_tokens": 5}, } ] ) @@ -406,10 +406,12 @@ def test_exact_entries_fill_only_missing_fields(restore_generalizations, monkeyp full = litellm.get_model_info("acme-full", custom_llm_provider="openai") assert full["supports_reasoning"] is False + assert full["max_output_tokens"] == 7 assert full["max_tokens"] == 7 bare = litellm.get_model_info("acme-bare", custom_llm_provider="openai") assert bare["supports_reasoning"] is True + assert bare["max_output_tokens"] == 5 assert bare["max_tokens"] == 5 assert bare["input_cost_per_token"] == 3e-6 assert bare["key"] == "acme-bare" @@ -525,7 +527,7 @@ def test_shipped_gemini_chat_baseline_loses_to_exact_entries(shipped_cost_map): model = "gemini-2.5-flash-lite" info = litellm.get_model_info(model, custom_llm_provider="gemini") entry = litellm.model_cost["gemini/gemini-2.5-flash-lite"] - assert info["max_tokens"] == entry["max_tokens"] + assert info["max_tokens"] == entry["max_output_tokens"] assert info["input_cost_per_token"] == entry["input_cost_per_token"] assert entry["input_cost_per_token"] > 0 diff --git a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py index 57e66805678..e9e3f7b8174 100644 --- a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py @@ -1,5 +1,9 @@ +from collections.abc import Iterator +from pathlib import Path from typing import Final +import respx + import asyncio, httpx, importlib, os import pytest @@ -8,6 +12,7 @@ from litellm import CustomLLM from litellm.litellm_core_utils import get_llm_provider_logic from litellm.litellm_core_utils.get_llm_provider_logic import ( get_llm_provider, + resolve_model_and_provider_for_metadata, get_provider_affinity_header, inferred_provider, is_registered_custom_provider, @@ -715,3 +720,47 @@ def test_opencode_affinity_resolution_preserves_existing_transport(prefix: str) model, provider, _, destination = get_llm_provider(model=f"{prefix}/test-model", api_base=api_base) assert (model, provider, destination) == ("test-model", prefix, api_base) assert get_provider_affinity_header(destination) == "x-opencode-session" + + +@pytest.fixture +def chatgpt_login_tripwire(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Iterator[None]: + blocked: Final = tmp_path / "blocked" + blocked.write_text("ordinary file") + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(blocked / "unused")) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + with respx.mock as transport: + yield + assert len(transport.calls) == 0 + + +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expected"), + [ + ("chatgpt/gpt-5.4", None, ("gpt-5.4", "chatgpt")), + ("gpt-5.4", "chatgpt", ("gpt-5.4", "chatgpt")), + ("github_copilot/gpt-4o", None, ("gpt-4o", "github_copilot")), + ("openai/gpt-5.4", None, ("gpt-5.4", "openai")), + ], +) +def test_resolve_model_and_provider_for_metadata_never_logs_in( + chatgpt_login_tripwire: None, model: str, custom_llm_provider: str | None, expected: tuple[str, str] +) -> None: + assert resolve_model_and_provider_for_metadata(model, custom_llm_provider) == expected + + +def test_metadata_callers_never_log_in_for_a_declared_chatgpt_deployment( + chatgpt_login_tripwire: None, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base + from litellm.proxy.spend_tracking.savings import _resolve_model + from litellm.router import Router + from litellm.types.router import LiteLLM_Params + from litellm.utils import supports_native_streaming + + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + assert supports_native_streaming("chatgpt/gpt-5.4", None) is True + assert get_api_base("chatgpt/gpt-5.4", {"model": "chatgpt/gpt-5.4"}) == "https://chatgpt.com/backend-api/codex" + identity = _resolve_model("chatgpt/gpt-5.4", None) + assert identity is not None and identity.provider == "chatgpt" + assert Router._model_group_llm_provider(LiteLLM_Params(model="chatgpt/gpt-5.4")) == ("gpt-5.4", "chatgpt") diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index c13cea51183..cc1fa1b8803 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -1,6 +1,7 @@ import asyncio import datetime import json +import queue import sys import threading import time @@ -2819,6 +2820,123 @@ def test_raise_on_model_repetition_tolerates_empty_choices( wrapper.raise_on_model_repetition() +_STUCK_CHUNK: Final = "the model is stuck on this" +_STUCK_PROVIDER_CHUNK_CAP: Final = 10 * litellm.REPEATED_STREAMING_CHUNK_LIMIT + + +class _StuckProvider: + def __init__(self) -> None: + self.closed_early = threading.Event() + self.chunks_sent = 0 + + def _next_chunk(self) -> ModelResponseStream: + self.chunks_sent += 1 + return _make_chunk(_STUCK_CHUNK) + + def sync_stream(self) -> Iterator[ModelResponseStream]: + try: + yield from (self._next_chunk() for _ in range(_STUCK_PROVIDER_CHUNK_CAP)) + except GeneratorExit: + self.closed_early.set() + raise + + async def async_stream(self) -> AsyncIterator[ModelResponseStream]: + try: + for _ in range(_STUCK_PROVIDER_CHUNK_CAP): + yield self._next_chunk() + except GeneratorExit: + self.closed_early.set() + raise + + +class _StuckStreamFailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failures: Final = queue.SimpleQueue[str]() + + def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.failures.put(str(cast(Mapping[str, object], kwargs)["exception"])) + + async def async_log_failure_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.failures.put(str(cast(Mapping[str, object], kwargs)["exception"])) + + +def _stuck_stream_wrapper( + provider_stream: Iterator[ModelResponseStream] | AsyncIterator[ModelResponseStream], + call_type: Literal["completion", "acompletion"], + recorder: _StuckStreamFailureRecorder, +) -> CustomStreamWrapper: + logging_obj: Final = Logging( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type=call_type, + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id=f"stuck-stream-{call_type}", + function_id=f"stuck-stream-{call_type}", + dynamic_failure_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="test-model", + optional_params={}, + litellm_params=get_litellm_params(custom_llm_provider="openai", acompletion=call_type == "acompletion"), + ) + return CustomStreamWrapper( + completion_stream=provider_stream, + model="test-model", + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + + +@pytest.mark.asyncio +async def test_async_stream_stuck_on_one_chunk_closes_the_provider_stream_and_logs_one_failure(): + """https://github.com/BerriAI/litellm/issues/19428 and https://github.com/BerriAI/litellm/issues/13786""" + provider: Final = _StuckProvider() + recorder: Final = _StuckStreamFailureRecorder() + wrapper: Final = _stuck_stream_wrapper(provider.async_stream(), "acompletion", recorder) + + with pytest.raises(Exception, match="repeating the same chunk"): + async for _ in wrapper: + pass + assert provider.closed_early.is_set() + assert provider.chunks_sent <= litellm.REPEATED_STREAMING_CHUNK_LIMIT + 1 + chunks_sent_at_failure: Final = provider.chunks_sent + + with pytest.raises(StopAsyncIteration): + await wrapper.__anext__() + await wrapper.wait_for_failure_logging(timeout=5) + + assert provider.chunks_sent == chunks_sent_at_failure + assert "repeating the same chunk" in recorder.failures.get_nowait() + assert recorder.failures.empty() + + +def test_sync_stream_stuck_on_one_chunk_closes_the_provider_stream_and_logs_one_failure(): + """Sync counterpart of the async test above""" + provider: Final = _StuckProvider() + recorder: Final = _StuckStreamFailureRecorder() + wrapper: Final = _stuck_stream_wrapper(provider.sync_stream(), "completion", recorder) + + with pytest.raises(Exception, match="repeating the same chunk"): + for _ in wrapper: + pass + assert provider.closed_early.is_set() + assert provider.chunks_sent <= litellm.REPEATED_STREAMING_CHUNK_LIMIT + 1 + chunks_sent_at_failure: Final = provider.chunks_sent + assert "repeating the same chunk" in recorder.failures.get(timeout=5) + + with pytest.raises(StopIteration): + next(wrapper) + + assert provider.chunks_sent == chunks_sent_at_failure + with pytest.raises(queue.Empty): + recorder.failures.get(timeout=0.5) + + def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj): """ Test that provider-reported usage from a post-finish_reason chunk diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 32f18b952ac..b3b04440d16 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -31,7 +31,7 @@ from litellm.llms.anthropic.pass_through.messages.transformation import ( ) from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig from litellm.llms.bedrock.claude_platform.transformation import BedrockClaudePlatformConfig -from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( +from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( @@ -2549,18 +2549,6 @@ def test_translate_system_message_preserves_cache_control(): # ============ Dynamic max_tokens Tests ============ -def test_get_max_tokens_for_model_claude_3(): - """ - Test that get_max_tokens_for_model returns correct value for Claude 3 models. - Claude 3 models have max_output_tokens of 4096. - """ - config = AnthropicConfig() - - # Claude 3 Sonnet should return 4096 - max_tokens = config.get_max_tokens_for_model("claude-3-sonnet-20240229") - assert max_tokens == 4096 - - def test_get_max_tokens_for_model_claude_35(): """ Test that get_max_tokens_for_model returns correct value for Claude 3.5 models. @@ -2581,24 +2569,24 @@ def test_get_max_tokens_for_model_claude_35(): def test_get_max_tokens_for_model_unknown(): """ - Test that get_max_tokens_for_model returns 4096 fallback for unknown models. + Test that get_max_tokens_for_model returns 128000 fallback for unknown models. """ config = AnthropicConfig() - # Unknown model should return 4096 as fallback + # Unknown model should return 128000 as fallback max_tokens = config.get_max_tokens_for_model("unknown-model-xyz") - assert max_tokens == 4096 + assert max_tokens == 128000 def test_get_max_tokens_for_model_none(): """ - Test that get_max_tokens_for_model returns 4096 fallback when model is None. + Test that get_max_tokens_for_model returns 128000 fallback when model is None. """ config = AnthropicConfig() - # None model should return 4096 as fallback + # None model should return 128000 as fallback max_tokens = config.get_max_tokens_for_model(None) - assert max_tokens == 4096 + assert max_tokens == 128000 def test_get_config_with_model_uses_dynamic_max_tokens(): @@ -2639,10 +2627,10 @@ def test_get_config_with_model_uses_dynamic_max_tokens(): def test_get_config_without_model_uses_fallback(): """ - Test that get_config without model parameter uses 4096 fallback. + Test that get_config without model parameter uses 128000 fallback. """ config = AnthropicConfig.get_config() - assert config["max_tokens"] == 4096 + assert config["max_tokens"] == 128000 def test_get_config_does_not_leak_module_constants(): @@ -6145,7 +6133,7 @@ def test_anthropic_messages_request_strips_billing_header_for_minimax(): def test_translate_system_message_strips_billing_header_for_bedrock_invoke(): - from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) @@ -6174,12 +6162,12 @@ def test_translate_system_message_strips_billing_header_for_bedrock_invoke(): True, ), ( - "litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation", + "litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation", "AmazonAnthropicClaudeConfig", True, ), ( - "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation", + "litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation", "AmazonAnthropicClaudeMessagesConfig", True, ), diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_reasoning_effort_translation.py b/tests/unit/llms/anthropic/pass_through/messages/test_reasoning_effort_translation.py index c1295305c7a..284c6ea532d 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_reasoning_effort_translation.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_reasoning_effort_translation.py @@ -11,7 +11,7 @@ from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, ) -from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( +from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) diff --git a/tests/unit/llms/azure/chat/test_azure_base_model_routing.py b/tests/unit/llms/azure/chat/test_azure_base_model_routing.py index aae2587085f..d9607c6db8c 100644 --- a/tests/unit/llms/azure/chat/test_azure_base_model_routing.py +++ b/tests/unit/llms/azure/chat/test_azure_base_model_routing.py @@ -12,7 +12,7 @@ import openai import pytest import litellm -from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config +from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.chat.o_series_transformation import AzureOpenAIO1Config from litellm.utils import ProviderConfigManager, get_optional_params from tests.unit.llms.bedrock.slow_upstream import STREAM_TIMEOUT_SECONDS, slow_upstream_transport diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index f753fb6b8f9..53df6610913 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -11,7 +11,7 @@ sys.path.insert( import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY -from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config +from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig from litellm.utils import _invalidate_model_cost_lowercase_map, get_optional_params from datetime import datetime diff --git a/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py similarity index 99% rename from tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py rename to tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py index 80328022f61..a82dd98e31d 100644 --- a/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py @@ -5,7 +5,7 @@ import pytest import litellm from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map -from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config +from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config @pytest.fixture() diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py similarity index 99% rename from tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py rename to tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py index bd98e4eb4de..86c4b29eb53 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py @@ -14,7 +14,7 @@ from pydantic import BaseModel, ConfigDict # Ensure the project root is on the import path so `litellm` can be imported when # tests are executed from any working directory. import litellm -from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( +from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -188,7 +188,7 @@ def test_output_format_conversion_to_inline_schema(): Bedrock Invoke doesn't support the output_format parameter, so LiteLLM converts it by embedding the schema directly into the user message content. """ - from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) @@ -265,7 +265,7 @@ def test_output_format_conversion_with_string_content(): """ Test that output_format conversion works when message content is a string (not a list). """ - from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) @@ -313,7 +313,7 @@ def test_output_format_with_no_schema(): """ Test that if output_format has no schema, the conversion is skipped gracefully. """ - from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) @@ -352,7 +352,7 @@ def test_opus_4_5_model_detection(): Test that the _is_claude_opus_4_5 method correctly identifies Opus 4.5 models with various naming conventions. """ - from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) @@ -1526,7 +1526,7 @@ class TestBedrockAnthropicPromptCachingRegression: from litellm.llms.bedrock.chat.converse_transformation import ( AmazonConverseConfig, ) - from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) @@ -1610,7 +1610,7 @@ class TestBedrockAnthropicPromptCachingRegression: from litellm.llms.bedrock.chat.converse_transformation import ( AmazonConverseConfig, ) - from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) @@ -1688,7 +1688,7 @@ class TestBedrockAnthropic1MContextRegression: from litellm.llms.bedrock.chat.converse_transformation import ( AmazonConverseConfig, ) - from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) @@ -1753,7 +1753,7 @@ class TestBedrockAnthropic1MContextRegression: from litellm.llms.bedrock.chat.converse_transformation import ( AmazonConverseConfig, ) - from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) @@ -1807,7 +1807,7 @@ class TestBedrockAnthropic1MContextRegression: from litellm.llms.bedrock.chat.converse_transformation import ( AmazonConverseConfig, ) - from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) @@ -1867,7 +1867,7 @@ class TestBedrockAnthropicCombinedRegressions: from litellm.llms.bedrock.chat.converse_transformation import ( AmazonConverseConfig, ) - from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py similarity index 99% rename from tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py rename to tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index 6fe32785281..4d9feea4454 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -37,7 +37,7 @@ from litellm.llms.bedrock.common_utils import ( normalize_custom_field_on_tools, normalize_tool_input_schema_types_for_bedrock_invoke, ) -from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( +from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, AmazonAnthropicClaudeMessagesStreamDecoder, ) diff --git a/tests/unit/llms/bedrock/test_anthropic_beta_support.py b/tests/unit/llms/bedrock/test_anthropic_beta_support.py index a20ec94a99d..73d529ae3ba 100644 --- a/tests/unit/llms/bedrock/test_anthropic_beta_support.py +++ b/tests/unit/llms/bedrock/test_anthropic_beta_support.py @@ -11,11 +11,11 @@ from unittest.mock import MagicMock, patch import pytest from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig -from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( +from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) from litellm.llms.bedrock.common_utils import get_anthropic_beta_from_headers -from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( +from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) diff --git a/tests/unit/llms/bedrock/test_base_aws_llm.py b/tests/unit/llms/bedrock/test_base_aws_llm.py index 4350fdb699b..c220e39f4d1 100644 --- a/tests/unit/llms/bedrock/test_base_aws_llm.py +++ b/tests/unit/llms/bedrock/test_base_aws_llm.py @@ -3228,7 +3228,7 @@ class TestGetBedrockModelIdArnHandling: def test_arn_url_matches_expected(self): """Full URL built from messages config must match expected encoded form.""" import urllib.parse - from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) diff --git a/tests/unit/llms/bedrock/test_request_metadata.py b/tests/unit/llms/bedrock/test_request_metadata.py index 6b74ad90cff..8f94ad54c3d 100644 --- a/tests/unit/llms/bedrock/test_request_metadata.py +++ b/tests/unit/llms/bedrock/test_request_metadata.py @@ -15,7 +15,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformati from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, ) -from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( +from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) from litellm.llms.bedrock.request_metadata import ( diff --git a/tests/unit/llms/chatgpt/chat/test_chatgpt_chat_transformation.py b/tests/unit/llms/chatgpt/chat/test_chatgpt_chat_transformation.py new file mode 100644 index 00000000000..f9d1dc28d12 --- /dev/null +++ b/tests/unit/llms/chatgpt/chat/test_chatgpt_chat_transformation.py @@ -0,0 +1,638 @@ +from unittest.mock import MagicMock + +import pytest + +from litellm.exceptions import AuthenticationError +from litellm.llms.chatgpt.chat.transformation import ChatGPTConfig +from litellm.llms.chatgpt.common_utils import GetAccessTokenError + +GATEWAY_BASE = "https://chatgpt.example/backend-api/codex" + + +def _gateway_authenticator() -> MagicMock: + authenticator = MagicMock() + authenticator.get_api_base.return_value = GATEWAY_BASE + authenticator.get_access_token.return_value = "gateway-access-token" + return authenticator + + +def _provider_info( + config: ChatGPTConfig, api_base: str | None, api_key: str | None +) -> tuple[str | None, str | None, str]: + return config._get_openai_compatible_provider_info( + model="gpt-5.4", api_base=api_base, api_key=api_key, custom_llm_provider="chatgpt" + ) + + +class TestChatGPTChatProviderInfo: + def test_configured_key_uses_configured_base_without_gateway_login(self): + authenticator = _gateway_authenticator() + config = ChatGPTConfig(authenticator=authenticator) + + assert _provider_info(config, "http://127.0.0.1:9999", "configured-key") == ( + "http://127.0.0.1:9999", + "configured-key", + "chatgpt", + ) + authenticator.get_access_token.assert_not_called() + + def test_configured_key_without_base_uses_gateway_base(self): + config = ChatGPTConfig(authenticator=_gateway_authenticator()) + + assert _provider_info(config, None, "configured-key") == (GATEWAY_BASE, "configured-key", "chatgpt") + + def test_gateway_login_is_never_sent_to_a_custom_base(self): + config = ChatGPTConfig(authenticator=_gateway_authenticator()) + + assert _provider_info(config, "http://attacker.example", None) == ( + GATEWAY_BASE, + "gateway-access-token", + "chatgpt", + ) + + def test_missing_gateway_login_raises_authentication_error(self): + authenticator = _gateway_authenticator() + authenticator.get_access_token.side_effect = GetAccessTokenError(status_code=401, message="not logged in") + config = ChatGPTConfig(authenticator=authenticator) + + with pytest.raises(AuthenticationError, match="not logged in"): + _provider_info(config, None, None) + + +import asyncio +import json +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import respx + +import litellm +from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.chatgpt.chat.transformation import ChatGPTConfig +from litellm.types.router import GenericLiteLLMParams + + +def _write_auth_record(path: Path, token: str, account_id: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "access_token": token, + "account_id": account_id, + "expires_at": 4102444800, + } + ) + ) + + +@pytest.fixture +def auth_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path, Path]: + monkeypatch.setattr("time.time", lambda: 1700000000) + default_dir: Final = tmp_path / "default" + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(default_dir)) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + _write_auth_record(default_dir / "auth.json", "token-default", "acct-default") + file_a: Final = tmp_path / "account-a" / "auth.json" + file_b: Final = tmp_path / "account-b" / "auth.json" + _write_auth_record(file_a, "token-a", "acct-a") + _write_auth_record(file_b, "token-b", "acct-b") + return file_a, file_b + + +class TestChatGPTMultiAccountChat: + def test_validate_environment_uses_per_model_auth_file(self, auth_files: tuple[Path, Path]) -> None: + file_a, file_b = auth_files + config: Final = ChatGPTConfig() + + headers_a: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + messages=[], + optional_params={}, + litellm_params={"chatgpt_auth_file": str(file_a)}, + api_key="token-default", + ) + headers_b: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + messages=[], + optional_params={}, + litellm_params={"chatgpt_auth_file": str(file_b)}, + api_key="token-default", + ) + + assert headers_a["Authorization"] == "Bearer token-a" + assert headers_a["ChatGPT-Account-Id"] == "acct-a" + assert headers_b["Authorization"] == "Bearer token-b" + assert headers_b["ChatGPT-Account-Id"] == "acct-b" + + def test_validate_environment_defaults_to_global_auth_file(self, auth_files: tuple[Path, Path]) -> None: + config: Final = ChatGPTConfig() + + headers: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + messages=[], + optional_params={}, + litellm_params={}, + api_key="token-default", + ) + + assert headers["Authorization"] == "Bearer token-default" + assert headers["ChatGPT-Account-Id"] == "acct-default" + + def test_provider_info_uses_per_model_auth_file(self, auth_files: tuple[Path, Path]) -> None: + file_a, _ = auth_files + config: Final = ChatGPTConfig() + + _, per_model_key, _ = config._get_openai_compatible_provider_info( + "gpt-6.1-sol", None, None, "chatgpt", {"chatgpt_auth_file": str(file_a)} + ) + _, default_key, _ = config._get_openai_compatible_provider_info("gpt-6.1-sol", None, None, "chatgpt") + + assert per_model_key == "token-a" + assert default_key == "token-default" + + def test_get_llm_provider_threads_auth_file(self, auth_files: tuple[Path, Path]) -> None: + file_a, _ = auth_files + + model, provider, dynamic_api_key, _ = get_llm_provider( + model="chatgpt/gpt-6.1-sol", + litellm_params=GenericLiteLLMParams(chatgpt_auth_file=str(file_a)), + ) + + assert model == "gpt-6.1-sol" + assert provider == "chatgpt" + assert dynamic_api_key == "token-a" + + +def _subscription_response(request: httpx.Request) -> httpx.Response: + account: Final = request.headers["ChatGPT-Account-Id"] + assert request.headers["Authorization"] in ( + f"Bearer token-{account.removeprefix('acct-')}", + f"Bearer {_client_token(account)}", + ) + assert "chatgpt_auth_file" not in request.content.decode() + message: Final = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": account, "annotations": []}], + } + response: Final = { + "id": "resp_test", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-6.1-sol", + "output": [message], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + events: Final = ( + {"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}, + { + "type": "response.output_text.delta", + "item_id": "msg_test", + "output_index": 0, + "content_index": 0, + "delta": account, + }, + {"type": "response.output_item.done", "output_index": 0, "item": message}, + {"type": "response.completed", "response": response}, + ) + return httpx.Response( + 200, + text="".join(f"data: {json.dumps(event)}\n\n" for event in events), + headers={"content-type": "text/event-stream"}, + ) + + +@pytest.mark.parametrize("blocked_default", [False, True]) +@pytest.mark.parametrize("with_prompt", [False, True]) +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("through_router", [False, True]) +@pytest.mark.asyncio +async def test_requests_use_selected_subscription( + auth_files: tuple[Path, Path], + endpoint: str, + through_router: bool, + with_prompt: bool, + blocked_default: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + default_dir: Final = auth_files[0].parent / "missing-default" + if blocked_default: + default_dir.write_text("ordinary file") + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(default_dir / "unused")) + if with_prompt: + monkeypatch.setattr( + litellm, + "callbacks", + [ + DotpromptManager( + prompt_id="account-check", + prompt_data={"content": "Saved account prompt", "metadata": {"model": "chatgpt/gpt-6.1-sol"}}, + ) + ], + ) + router: Final = ( + litellm.Router( + model_list=[ + { + "model_name": f"account-{index}", + "litellm_params": {"model": "chatgpt/gpt-6.1-sol", "chatgpt_auth_file": str(path)}, + } + for index, path in enumerate(auth_files) + ] + ) + if through_router + else None + ) + + async def request(index: int, path: Path) -> str: + params: Final = MappingProxyType( + { + "model": f"account-{index}" if router else "chatgpt/gpt-6.1-sol", + **({"chatgpt_auth_file": str(path)} if router is None else {}), + **({"prompt_id": "account-check"} if with_prompt else {}), + } + ) + if endpoint == "chat": + result: Final = await (router.acompletion if router else litellm.acompletion)( + **params, messages=[{"role": "user", "content": "Hello"}] + ) + return result.choices[0].message.content + response: Final = await (router.aresponses if router else litellm.aresponses)( + **params, input=[{"role": "user", "content": "Hello"}] + ) + return response.output[0].content[0].text + + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").mock(side_effect=_subscription_response) + results: Final = await asyncio.gather(*(request(index, path) for index, path in enumerate(auth_files))) + assert results == ["acct-a", "acct-b"] + if with_prompt: + assert all("Saved account prompt" in call.request.content.decode() for call in transport.calls) + + +def _client_token(account: str) -> str: + import base64 + + payload: Final = ( + base64.urlsafe_b64encode(json.dumps({"https://api.openai.com/auth": {"chatgpt_account_id": account}}).encode()) + .rstrip(b"=") + .decode() + ) + return f"header.{payload}.signature" + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("configured_login", [False, True]) +@pytest.mark.parametrize("with_prompt", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.asyncio +async def test_client_login_keeps_its_account_across_gateway_deployments( + auth_files: tuple[Path, Path], + endpoint: str, + configured_login: bool, + with_prompt: bool, + stream: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(auth_files[0].parent / "no-gateway-login")) + if with_prompt: + monkeypatch.setattr( + litellm, + "callbacks", + [ + DotpromptManager( + prompt_id="client-account-check", + prompt_data={"content": "Saved client prompt", "metadata": {"model": "chatgpt/gpt-6.1-sol"}}, + ) + ], + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "subscriptions", + "litellm_params": { + "model": "chatgpt/gpt-6.1-sol", + **({"chatgpt_auth_file": str(path)} if configured_login else {}), + }, + } + for path in auth_files + ], + num_retries=0, + ) + + def reply(request: httpx.Request) -> httpx.Response: + account: Final = request.headers["ChatGPT-Account-Id"] + assert request.headers["Authorization"] == f"Bearer {_client_token(account)}" + return _subscription_response( + httpx.Request( + "POST", + request.url, + content=request.content, + headers={ + **{key: value for key, value in request.headers.items() if key.lower() != "authorization"}, + "Authorization": f"Bearer token-{account.removeprefix('acct-')}", + }, + ) + ) + + async def call(account: str) -> str: + params: Final = MappingProxyType( + { + "model": "subscriptions", + "stream": stream, + **({"prompt_id": "client-account-check"} if with_prompt else {}), + "provider_specific_header": { + "custom_llm_provider": "chatgpt", + "extra_headers": {"Authorization": f"Bearer {_client_token(account)}"}, + }, + } + ) + response: Final = await ( + router.acompletion(**params, messages=[{"role": "user", "content": "Hello"}]) + if endpoint == "chat" + else router.aresponses(**params, input=[{"role": "user", "content": "Hello"}]) + ) + if stream: + chunks: Final = [part async for part in response] + return ( + "".join(part.choices[0].delta.content or "" for part in chunks) + if endpoint == "chat" + else "".join( + getattr(part, "delta", "") + for part in chunks + if getattr(part, "type", "") == "response.output_text.delta" + ) + ) + return response.choices[0].message.content if endpoint == "chat" else response.output[0].content[0].text + + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").mock(side_effect=reply) + results: Final = await asyncio.gather(call("acct-client-a"), call("acct-client-b")) + assert results == ["acct-client-a", "acct-client-b"] + assert len(transport.calls) == 2 + + +@pytest.mark.asyncio +async def test_one_client_limit_does_not_cool_down_other_subscriptions( + auth_files: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + {"model_name": "pool", "litellm_params": {"model": "chatgpt/gpt-6.1-sol", "chatgpt_auth_file": str(path)}} + for path in auth_files + ], + num_retries=0, + allowed_fails=0, + cooldown_time=60, + ) + headers: Final = { + "custom_llm_provider": "chatgpt", + "extra_headers": {"Authorization": f"Bearer {_client_token('acct-limited')}"}, + } + with respx.mock as transport: + route: Final = transport.post("https://chatgpt.com/backend-api/codex/responses") + route.respond( + 429, + headers={"retry-after": "60"}, + json={"error": {"message": "Usage limit reached", "type": "rate_limit_error"}}, + ) + for _ in range(2): + with pytest.raises(litellm.RateLimitError): + await router.aresponses(model="pool", input="Hello", provider_specific_header=headers) + await asyncio.sleep(0) + route.mock(side_effect=_subscription_response) + response: Final = await router.aresponses(model="pool", input="Hello") + assert response.output[0].content[0].text in ("acct-a", "acct-b") + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("async_call", [False, True]) +@pytest.mark.parametrize("direct_headers", [False, True]) +@pytest.mark.parametrize("configured_key", [False, True]) +@pytest.mark.parametrize( + "target", ["https://example.com/v1", "http://chatgpt.com/backend-api/codex", "https://chatgpt.com.example.com/v1"] +) +@pytest.mark.asyncio +async def test_client_login_cannot_be_redirected_to_another_host( + endpoint: str, target: str, async_call: bool, direct_headers: bool, configured_key: bool +) -> None: + headers: Final = {"Authorization": f"Bearer {_client_token('acct-client')}"} + credential: Final = ( + {"extra_headers": headers} + if direct_headers + else {"provider_specific_header": {"custom_llm_provider": "chatgpt", "extra_headers": headers}} + ) + params: Final = MappingProxyType( + { + "model": "chatgpt/gpt-6.1-sol", + "api_base": target, + **({"api_key": "configured-test-key"} if configured_key else {}), + **credential, + } + ) + + async def request() -> None: + if endpoint == "chat": + if async_call: + await litellm.acompletion(**params, messages=[{"role": "user", "content": "Hello"}]) + else: + litellm.completion(**params, messages=[{"role": "user", "content": "Hello"}]) + elif async_call: + await litellm.aresponses(**params, input="Hello") + else: + litellm.responses(**params, input="Hello") + + with respx.mock as transport: + with pytest.raises(litellm.BadRequestError, match="official HTTPS ChatGPT endpoint"): + await request() + assert len(transport.calls) == 0 + + +@pytest.mark.asyncio +async def test_cross_provider_fallback_does_not_receive_the_chatgpt_login( + auth_files: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "subscription", + "litellm_params": {"model": "chatgpt/gpt-6.1-sol", "chatgpt_auth_file": str(auth_files[0])}, + }, + { + "model_name": "api-fallback", + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "sk-example-fallback"}, + }, + ], + fallbacks=[{"subscription": ["api-fallback"]}], + num_retries=0, + ) + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").respond( + 429, json={"error": {"message": "Usage limit reached", "type": "rate_limit_error"}} + ) + fallback: Final = transport.post("https://api.openai.com/v1/responses").respond( + 200, + json={ + "id": "resp_fallback", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-6.1-sol", + "output": [ + { + "id": "msg_fallback", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "fallback", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + response: Final = await router.aresponses( + model="subscription", + input="Hello", + provider_specific_header={ + "custom_llm_provider": "chatgpt", + "extra_headers": { + "Authorization": f"Bearer {_client_token('acct-client')}", + "ChatGPT-Account-Id": "acct-client", + }, + }, + ) + assert response.output[0].content[0].text == "fallback" + assert fallback.calls.last.request.headers["Authorization"] == "Bearer sk-example-fallback" + assert "ChatGPT-Account-Id" not in fallback.calls.last.request.headers + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("async_call", [False, True]) +@pytest.mark.parametrize("with_prompt", [False, True]) +@pytest.mark.asyncio +async def test_sdk_extra_headers_use_client_login_without_gateway_auth( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, endpoint: str, async_call: bool, with_prompt: bool +) -> None: + blocked: Final = tmp_path / "blocked" + blocked.write_text("ordinary file") + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(blocked / "unused")) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + headers: Final = {"Authorization": f"Bearer {_client_token('acct-client-a')}"} + prompt_params: Final = {"prompt_id": "sdk-client"} if with_prompt else {} + if with_prompt: + monkeypatch.setattr( + litellm, + "callbacks", + [ + DotpromptManager( + prompt_id="sdk-client", + prompt_data={"content": "Hello", "metadata": {"model": "chatgpt/gpt-6-luna"}}, + ) + ], + ) + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").mock(side_effect=_subscription_response) + if endpoint == "chat": + params: Final = { + "model": "chatgpt/gpt-6.1-sol", + "messages": [{"role": "user", "content": "Hello"}], + "extra_headers": headers, + **prompt_params, + } + result: Final = await litellm.acompletion(**params) if async_call else litellm.completion(**params) + assert result.choices[0].message.content == "acct-client-a" + else: + response: Final = ( + await litellm.aresponses( + model="chatgpt/gpt-6.1-sol", input="Hello", extra_headers=headers, **prompt_params + ) + if async_call + else litellm.responses( + model="chatgpt/gpt-6.1-sol", input="Hello", extra_headers=headers, **prompt_params + ) + ) + assert response.output[0].content[0].text == "acct-client-a" + assert transport.calls.last.request.headers["Authorization"] == headers["Authorization"] + assert len(transport.calls) == 1 + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("sync", [False, True]) +@pytest.mark.parametrize("credential_source", ["default", "file", "api_key"]) +@pytest.mark.parametrize("authorization_header", [None, "Authorization", "authorization"]) +@pytest.mark.asyncio +async def test_requests_preserve_gateway_token_and_selected_workspace( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + endpoint: str, + sync: bool, + credential_source: str, + authorization_header: str | None, +) -> None: + monkeypatch.setattr("time.time", lambda: 1700000000) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("CHATGPT_AUTH_FILE", str(tmp_path / "default.json")) + token: Final = _client_token("acct-token-default") + auth_file: Final = tmp_path / ("selected.json" if credential_source == "file" else "default.json") + _write_auth_record(auth_file, token, "acct-saved-workspace") + configured_token: Final = _client_token("acct-configured") + expected_token: Final = configured_token if credential_source == "api_key" else token + expected_account: Final = "acct-configured" if credential_source == "api_key" else "acct-saved-workspace" + + def reply(request: httpx.Request) -> httpx.Response: + assert request.headers.get_list("authorization") == [f"Bearer {expected_token}"] + assert request.headers.get_list("chatgpt-account-id") == [expected_account] + assert request.headers["originator"] == "codex_cli_rs" + return _subscription_response( + httpx.Request( + "POST", + request.url, + content=request.content, + headers={ + "Authorization": f"Bearer token-{expected_account.removeprefix('acct-')}", + "ChatGPT-Account-Id": expected_account, + }, + ) + ) + + params: Final = MappingProxyType( + { + "model": "chatgpt/gpt-6.1-sol", + **({"chatgpt_auth_file": str(auth_file)} if credential_source == "file" else {}), + **({"api_key": configured_token} if credential_source == "api_key" else {}), + "extra_headers": { + "originator": "codex_cli_rs", + **({authorization_header: "Bearer sk-unrelated-example"} if authorization_header else {}), + }, + } + ) + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").mock(side_effect=reply) + if endpoint == "chat": + result: Final = ( + await asyncio.to_thread(litellm.completion, **params, messages=[{"role": "user", "content": "Hello"}]) + if sync + else await litellm.acompletion(**params, messages=[{"role": "user", "content": "Hello"}]) + ) + assert result.choices[0].message.content == expected_account + else: + response: Final = ( + await asyncio.to_thread(litellm.responses, **params, input=[{"role": "user", "content": "Hello"}]) + if sync + else await litellm.aresponses(**params, input=[{"role": "user", "content": "Hello"}]) + ) + assert response.output[0].content[0].text == expected_account diff --git a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index e47d7fc702c..d88bdfc0b41 100644 --- a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -4,8 +4,10 @@ Tests for ChatGPT subscription Responses API transformation Source: litellm/llms/chatgpt/responses/transformation.py """ +import base64 import json from collections.abc import Generator +from pathlib import Path from typing import Final from unittest.mock import MagicMock, patch @@ -13,6 +15,7 @@ import httpx import pytest import litellm +from litellm.llms.chatgpt.authenticator import Authenticator from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig from litellm.llms.openai.common_utils import OpenAIError from litellm.main import responses_api_bridge_check @@ -395,3 +398,165 @@ class TestChatGPTResponsesAPITransformation: assert exc_info.value.message == "ChatGPT upstream failed" assert exc_info.value.status_code == 502 + + +def _fake_chatgpt_oauth_token(account_id: str) -> str: + header = base64.urlsafe_b64encode(b'{"alg":"RS256"}').rstrip(b"=").decode() + payload = ( + base64.urlsafe_b64encode( + json.dumps({"https://api.openai.com/auth": {"chatgpt_account_id": account_id}}).encode() + ) + .rstrip(b"=") + .decode() + ) + return f"{header}.{payload}.fake-signature" + + +def _gateway_authenticator() -> MagicMock: + authenticator = MagicMock() + authenticator.get_access_token.return_value = "gateway-access-token" + authenticator.get_account_id.return_value = "acct-gateway" + return authenticator + + +def _config_with(authenticator: MagicMock) -> ChatGPTResponsesAPIConfig: + return ChatGPTResponsesAPIConfig(authenticator=authenticator) + + +class TestChatGPTClientCredentialForwarding: + def test_client_chatgpt_oauth_wins_over_configured_key_and_gateway_login(self): + authenticator = _gateway_authenticator() + token = _fake_chatgpt_oauth_token("acct-alice") + + headers = _config_with(authenticator).validate_environment( + headers={"Authorization": f"Bearer {token}", "originator": "codex_cli_rs"}, + model="gpt-5.4", + litellm_params=GenericLiteLLMParams(api_key="configured-gateway-key", litellm_session_id="session-1"), + ) + + assert headers["Authorization"] == f"Bearer {token}" + assert headers["ChatGPT-Account-Id"] == "acct-alice" + assert headers["originator"] == "codex_cli_rs" + assert headers["accept"] == "text/event-stream" + assert [name for name in headers if name.lower() == "authorization"] == ["Authorization"] + authenticator.get_access_token.assert_not_called() + + def test_explicit_account_header_wins_over_jwt_claim(self): + token = _fake_chatgpt_oauth_token("acct-from-jwt") + + headers = _config_with(_gateway_authenticator()).validate_environment( + headers={"authorization": f"Bearer {token}", "chatgpt-account-id": "acct-from-header"}, + model="gpt-5.4", + litellm_params=GenericLiteLLMParams(), + ) + + assert headers["ChatGPT-Account-Id"] == "acct-from-header" + assert [name for name in headers if name.lower() == "chatgpt-account-id"] == ["ChatGPT-Account-Id"] + + @pytest.mark.parametrize( + "bearer", ["Bearer sk-proj-fake-platform-key", "Bearer not-a-jwt", "Bearer a.bm90LWpzb24.c"] + ) + def test_non_chatgpt_bearer_falls_back_to_configured_key(self, bearer, tmp_path): + authenticator = Authenticator(auth_file=str(tmp_path / "missing-default.json")) + + headers = ChatGPTResponsesAPIConfig(authenticator=authenticator).validate_environment( + headers={"Authorization": bearer}, + model="gpt-5.4", + litellm_params=GenericLiteLLMParams(api_key="configured-gateway-key"), + ) + + assert "ChatGPT-Account-Id" not in headers + assert headers["Authorization"] == "Bearer configured-gateway-key" + + def test_configured_key_is_used_without_client_credential(self): + authenticator = _gateway_authenticator() + + headers = _config_with(authenticator).validate_environment( + headers={}, + model="gpt-5.4", + litellm_params=GenericLiteLLMParams(api_key="configured-gateway-key"), + ) + + assert headers["Authorization"] == "Bearer configured-gateway-key" + authenticator.get_access_token.assert_not_called() + + def test_gateway_login_is_used_without_client_credential_or_configured_key(self): + headers = _config_with(_gateway_authenticator()).validate_environment( + headers={}, + model="gpt-5.4", + litellm_params=GenericLiteLLMParams(), + ) + + assert headers["Authorization"] == "Bearer gateway-access-token" + assert headers["ChatGPT-Account-Id"] == "acct-gateway" + + def test_two_clients_never_share_a_credential(self): + config = _config_with(_gateway_authenticator()) + alice = _fake_chatgpt_oauth_token("acct-alice") + bob = _fake_chatgpt_oauth_token("acct-bob") + + forwarded = [ + config.validate_environment( + headers={"Authorization": f"Bearer {token}"}, model="gpt-5.4", litellm_params=GenericLiteLLMParams() + ) + for token in (alice, bob, alice) + ] + + assert [h["Authorization"] for h in forwarded] == [f"Bearer {alice}", f"Bearer {bob}", f"Bearer {alice}"] + assert [h["ChatGPT-Account-Id"] for h in forwarded] == ["acct-alice", "acct-bob", "acct-alice"] + + +class TestChatGPTResponsesMultiAccount: + @pytest.fixture(autouse=True) + def fixed_clock(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("time.time", lambda: 1700000000) + + @staticmethod + def _write_auth_record(path: Path, token: str, account_id: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "access_token": token, + "account_id": account_id, + "expires_at": 4102444800, + } + ) + ) + + def test_validate_environment_uses_per_model_auth_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + default_dir: Final = tmp_path / "default" + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(default_dir)) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + self._write_auth_record(default_dir / "auth.json", "token-default", "acct-default") + file_a: Final = tmp_path / "account-a" / "auth.json" + file_b: Final = tmp_path / "account-b" / "auth.json" + self._write_auth_record(file_a, "token-a", "acct-a") + self._write_auth_record(file_b, "token-b", "acct-b") + + config: Final = ChatGPTResponsesAPIConfig() + + headers_a: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + litellm_params=GenericLiteLLMParams(chatgpt_auth_file=str(file_a)), + ) + headers_b: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + litellm_params=GenericLiteLLMParams(chatgpt_auth_file=str(file_b)), + ) + headers_default: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + litellm_params=GenericLiteLLMParams(), + ) + + assert headers_a["Authorization"] == "Bearer token-a" + assert headers_a["ChatGPT-Account-Id"] == "acct-a" + assert headers_b["Authorization"] == "Bearer token-b" + assert headers_b["ChatGPT-Account-Id"] == "acct-b" + assert headers_default["Authorization"] == "Bearer token-default" + assert headers_default["ChatGPT-Account-Id"] == "acct-default" diff --git a/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py b/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py index 43a4c26b6d2..2e9e3462bee 100644 --- a/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py +++ b/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py @@ -2,17 +2,24 @@ import base64 import json import time from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final from unittest.mock import MagicMock, mock_open, patch import pytest +import httpx +import respx import litellm from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS from litellm.llms.chatgpt.authenticator import ( TOKEN_REFRESH_TIMEOUT_SECONDS, Authenticator, + get_cached_authenticator, + get_chatgpt_auth_file, ) from litellm.llms.chatgpt.common_utils import GetAccessTokenError +from litellm.types.router import GenericLiteLLMParams def _make_jwt(payload: dict) -> str: @@ -161,3 +168,144 @@ class TestChatGPTAuthenticator: assert account_id == "acct-123" mock_write.assert_called_once() assert mock_write.call_args[0][0]["account_id"] == "acct-123" + + def test_get_account_id_prefers_stored_value_without_rewriting_file(self, authenticator): + id_token = _make_jwt({"https://api.openai.com/auth": {"chatgpt_account_id": "acct-from-token"}}) + auth_data = json.dumps({"account_id": "acct-stored", "id_token": id_token}) + + with ( + patch("builtins.open", mock_open(read_data=auth_data)), + patch.object(authenticator, "_write_auth_file") as mock_write, + ): + assert authenticator.get_account_id() == "acct-stored" + mock_write.assert_not_called() + + def test_get_account_id_without_login_returns_none(self, authenticator): + with ( + patch("builtins.open", mock_open(read_data="{}")), + patch.object(authenticator, "_write_auth_file") as mock_write, + ): + assert authenticator.get_account_id() is None + mock_write.assert_not_called() + + +def _write_auth_record(path: Path, token: str, account_id: str) -> None: + path.write_text( + json.dumps( + { + "access_token": token, + "account_id": account_id, + "expires_at": 4102444800, + } + ) + ) + + +class TestChatGPTMultiAccountAuthenticator: + @pytest.fixture(autouse=True) + def fixed_clock(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("time.time", lambda: 1700000000) + + def test_explicit_auth_file_overrides_env(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "env-dir")) + monkeypatch.setenv("CHATGPT_AUTH_FILE", "env.json") + custom_file: Final = tmp_path / "account-a" / "auth.json" + + authenticator: Final = Authenticator(auth_file=str(custom_file)) + + assert authenticator.auth_file == str(custom_file) + assert authenticator.token_dir == str(custom_file.parent) + assert not custom_file.parent.exists() + authenticator._write_auth_file({"access_token": "token-a"}) + assert json.loads(custom_file.read_text())["access_token"] == "token-a" + + def test_default_auth_file_from_env(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "env-dir")) + monkeypatch.setenv("CHATGPT_AUTH_FILE", "env.json") + + authenticator: Final = Authenticator() + + assert authenticator.auth_file == str(tmp_path / "env-dir" / "env.json") + + def test_bare_filename_auth_file_resolves_against_cwd( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + + authenticator: Final = Authenticator(auth_file="auth.json") + + assert authenticator.auth_file == str(tmp_path / "auth.json") + assert authenticator.token_dir == str(tmp_path) + + def test_two_auth_files_are_isolated(self, tmp_path: Path) -> None: + file_a: Final = tmp_path / "account-a.json" + file_b: Final = tmp_path / "account-b.json" + _write_auth_record(file_a, "token-a", "acct-a") + _write_auth_record(file_b, "token-b", "acct-b") + + authenticator_a: Final = Authenticator(auth_file=str(file_a)) + authenticator_b: Final = Authenticator(auth_file=str(file_b)) + + assert authenticator_a.get_access_token() == "token-a" + assert authenticator_a.get_account_id() == "acct-a" + assert authenticator_b.get_access_token() == "token-b" + assert authenticator_b.get_account_id() == "acct-b" + + def test_get_cached_authenticator_reuses_instance_per_path(self, tmp_path: Path) -> None: + file_a: Final = tmp_path / "account-a.json" + file_b: Final = tmp_path / "account-b.json" + + authenticator_a: Final = get_cached_authenticator(str(file_a)) + + assert get_cached_authenticator(str(file_a)) is authenticator_a + assert get_cached_authenticator(str(file_b)) is not authenticator_a + + def test_get_chatgpt_auth_file(self) -> None: + assert get_chatgpt_auth_file(None) is None + assert get_chatgpt_auth_file({}) is None + assert get_chatgpt_auth_file({"chatgpt_auth_file": ""}) is None + assert get_chatgpt_auth_file({"chatgpt_auth_file": "/a/auth.json"}) == "/a/auth.json" + assert get_chatgpt_auth_file(GenericLiteLLMParams()) is None + assert get_chatgpt_auth_file(GenericLiteLLMParams(chatgpt_auth_file="/b/auth.json")) == "/b/auth.json" + + def test_relative_and_absolute_auth_file_share_cached_authenticator( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + get_cached_authenticator.cache_clear() + auth_file: Final = tmp_path / "x" / "auth.json" + auth_file.parent.mkdir(parents=True) + auth_file.write_text("{}") + + relative_key: Final = get_chatgpt_auth_file({"chatgpt_auth_file": "x/auth.json"}) + absolute_key: Final = get_chatgpt_auth_file({"chatgpt_auth_file": str(auth_file)}) + assert relative_key is not None and absolute_key is not None + + assert get_cached_authenticator(relative_key) is get_cached_authenticator(absolute_key) + + +def test_refresh_keeps_saved_workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("time.time", lambda: 1700000000) + auth_file: Final = tmp_path / "selected.json" + auth_file.write_text( + json.dumps( + { + "access_token": "expired-example-token", + "refresh_token": "example-refresh-token", + "expires_at": 1, + "account_id": "acct-selected-workspace", + } + ) + ) + token: Final = _make_jwt({"exp": 4102444800, "https://api.openai.com/auth": {"chatgpt_account_id": "acct-default"}}) + authenticator: Final = Authenticator(auth_file=str(auth_file)) + with respx.mock as transport: + transport.post("https://auth.openai.com/oauth/token").mock( + return_value=httpx.Response( + 200, + json={"access_token": token, "id_token": token, "refresh_token": "example-rotated-refresh"}, + ) + ) + assert authenticator.get_access_token() == token + assert authenticator.get_account_id(token) == "acct-selected-workspace" + assert json.loads(auth_file.read_text())["refresh_token"] == "example-rotated-refresh" diff --git a/tests/unit/llms/chatgpt/test_chatgpt_common_utils.py b/tests/unit/llms/chatgpt/test_chatgpt_common_utils.py new file mode 100644 index 00000000000..6b280d97089 --- /dev/null +++ b/tests/unit/llms/chatgpt/test_chatgpt_common_utils.py @@ -0,0 +1,90 @@ +import base64 +import json + +import pytest + +from litellm.llms.chatgpt.common_utils import ( + extract_chatgpt_account_id, + get_chatgpt_client_credential, + is_chatgpt_oauth_key, + select_chatgpt_client_credential_headers, +) + + +def _make_jwt(payload: dict) -> str: + def _b64(obj: dict) -> str: + raw = json.dumps(obj, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("utf-8").rstrip("=") + + return f"{_b64({'alg': 'none', 'typ': 'JWT'})}.{_b64(payload)}." + + +CHATGPT_JWT = _make_jwt({"https://api.openai.com/auth": {"chatgpt_account_id": "acct-alice"}}) +CHATGPT_JWT_WITHOUT_ACCOUNT = _make_jwt({"https://api.openai.com/auth": {"user_id": "user-1"}}) +NON_CHATGPT_JWT = _make_jwt({"sub": "someone", "https://api.openai.com/auth": "not-an-object"}) + + +class TestExtractChatGPTAccountId: + def test_reads_account_id_from_auth_claim(self): + assert extract_chatgpt_account_id(CHATGPT_JWT) == "acct-alice" + + @pytest.mark.parametrize( + "token", + [None, "", "sk-proj-not-a-jwt", "header.only", NON_CHATGPT_JWT, CHATGPT_JWT_WITHOUT_ACCOUNT], + ) + def test_returns_none_without_a_usable_account_claim(self, token): + assert extract_chatgpt_account_id(token) is None + + +class TestIsChatGPTOAuthKey: + def test_recognizes_bearer_and_raw_chatgpt_jwt(self): + assert is_chatgpt_oauth_key(CHATGPT_JWT) + assert is_chatgpt_oauth_key(f"Bearer {CHATGPT_JWT}") + assert is_chatgpt_oauth_key(f"Bearer {CHATGPT_JWT_WITHOUT_ACCOUNT}") + + @pytest.mark.parametrize( + "value", [None, "", "Bearer sk-proj-platform-key", "Bearer not.a.jwt", _make_jwt({"sub": "x"})] + ) + def test_rejects_non_chatgpt_credentials(self, value): + assert not is_chatgpt_oauth_key(value) + + +class TestSelectChatGPTClientCredentialHeaders: + def test_account_header_alone_is_not_forwarded(self): + selected = select_chatgpt_client_credential_headers( + {"chatgpt-account-id": "acct-alice", "authorization": "Bearer sk-proj-platform-key"} + ) + assert dict(selected) == {} + assert get_chatgpt_client_credential({"chatgpt-account-id": "acct-alice"}) is None + + def test_forwards_only_bearer_and_account_context(self): + selected = select_chatgpt_client_credential_headers( + { + "Authorization": f"Bearer {CHATGPT_JWT}", + "ChatGPT-Account-Id": "acct-explicit", + "x-litellm-api-key": "sk-gateway", + "user-agent": "codex", + } + ) + assert dict(selected) == {"Authorization": f"Bearer {CHATGPT_JWT}", "ChatGPT-Account-Id": "acct-explicit"} + credential = get_chatgpt_client_credential(selected) + assert credential is not None + assert credential.access_token == CHATGPT_JWT + assert credential.account_id == "acct-explicit" + + def test_account_id_falls_back_to_jwt_claim(self): + credential = get_chatgpt_client_credential({"authorization": f"Bearer {CHATGPT_JWT}"}) + assert credential is not None + assert credential.account_id == "acct-alice" + + +@pytest.mark.parametrize("scheme", ["Bearer", "bearer", "BEARER"]) +def test_bearer_scheme_is_case_insensitive(scheme): + credential = get_chatgpt_client_credential({"authorization": f"{scheme} {CHATGPT_JWT}"}) + assert credential is not None + assert credential.access_token == CHATGPT_JWT + assert credential.account_id == "acct-alice" + + +def test_other_authentication_schemes_are_not_chatgpt_logins(): + assert not is_chatgpt_oauth_key(f"Basic {CHATGPT_JWT}") diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index bd4ed4c99cc..1607447f7bf 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -2172,3 +2172,86 @@ async def test_async_ollama_ssl_verify_false_builds_a_client_that_skips_certific assert route.call_count == 1 assert text == "Sunny" assert _cached_clients_verify_modes(client_cache, AsyncHTTPHandler) == (ssl.CERT_NONE,) + + +_TIMEOUT_CLIENT_LIMITS: Final = httpx.Timeout(42.0, connect=7.0) +_TIMEOUT_PHASES: Final = ( + (httpx.ConnectTimeout, "Connection timed out", "Timeout passed=7.0"), + (httpx.ReadTimeout, "Read timed out", "Timeout passed=42.0"), + (httpx.WriteTimeout, "Write timed out", "Timeout passed=42.0"), + (httpx.PoolTimeout, "Connection pool timed out", "Timeout passed=42.0"), +) + + +def _timing_out(kind: type[httpx.TimeoutException]) -> httpx.MockTransport: + def _raise(request: httpx.Request) -> httpx.Response: + raise kind("simulated", request=request) + + return httpx.MockTransport(_raise) + + +@pytest.mark.parametrize("verb", ["post", "put", "patch", "delete"]) +@pytest.mark.parametrize(("kind", "label", "limit"), _TIMEOUT_PHASES) +def test_sync_timeout_without_a_per_call_timeout_names_the_phase_and_the_limit_that_fired( + verb: str, kind: type[httpx.TimeoutException], label: str, limit: str +) -> None: + """Issue #14635: the handler used to print the caller's absent argument as 'after None seconds'.""" + handler: Final = HTTPHandler() + handler.client.close() + handler.client = httpx.Client(transport=_timing_out(kind), timeout=_TIMEOUT_CLIENT_LIMITS) + try: + with pytest.raises(LitellmTimeout) as raised: + getattr(handler, verb)("https://example.com/v1/embeddings", json={"input": "hi"}) + finally: + handler.close() + + message: Final = str(raised.value) + assert label in message, message + assert limit in message, message + assert "None" not in message, message + + +@pytest.mark.parametrize("verb", ["post", "put", "patch"]) +@pytest.mark.parametrize(("kind", "label", "limit"), _TIMEOUT_PHASES) +async def test_async_timeout_without_a_per_call_timeout_names_the_phase_and_the_limit_that_fired( + verb: str, kind: type[httpx.TimeoutException], label: str, limit: str +) -> None: + handler: Final = AsyncHTTPHandler(timeout=_TIMEOUT_CLIENT_LIMITS) + await handler.client.aclose() + handler.client = httpx.AsyncClient(transport=_timing_out(kind), timeout=_TIMEOUT_CLIENT_LIMITS) + try: + with pytest.raises(LitellmTimeout) as raised: + await getattr(handler, verb)("https://example.com/v1/embeddings", json={"input": "hi"}) + finally: + await handler.close() + + message: Final = str(raised.value) + assert label in message, message + assert limit in message, message + assert "None" not in message, message + + +def test_sync_timeout_reports_the_per_call_timeout_as_a_float_when_one_is_given() -> None: + handler: Final = HTTPHandler() + handler.client.close() + handler.client = httpx.Client(transport=_timing_out(httpx.ReadTimeout), timeout=_TIMEOUT_CLIENT_LIMITS) + try: + with pytest.raises(LitellmTimeout) as raised: + handler.post("https://example.com/v1/embeddings", json={"input": "hi"}, timeout=30) + finally: + handler.close() + + assert "Read timed out. Timeout passed=30.0, time taken=" in str(raised.value), str(raised.value) + + +async def test_async_timeout_reports_the_per_call_timeout_as_a_float_when_one_is_given() -> None: + handler: Final = AsyncHTTPHandler(timeout=_TIMEOUT_CLIENT_LIMITS) + await handler.client.aclose() + handler.client = httpx.AsyncClient(transport=_timing_out(httpx.ReadTimeout), timeout=_TIMEOUT_CLIENT_LIMITS) + try: + with pytest.raises(LitellmTimeout) as raised: + await handler.post("https://example.com/v1/embeddings", json={"input": "hi"}, timeout=30) + finally: + await handler.close() + + assert "Read timed out. Timeout passed=30.0, time taken=" in str(raised.value), str(raised.value) diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index bbcd7f70e63..55fad96ceb1 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -49,7 +49,7 @@ from litellm.llms.custom_httpx.llm_http_handler import ( _rust_responses_websocket_enabled, ) from litellm.llms.azure.videos.transformation import AzureVideoConfig -from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( +from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) from litellm.llms.bedrock.files.transformation import BedrockFilesConfig @@ -3173,7 +3173,7 @@ async def test_anthropic_invalid_thinking_signature_retry_resigns_bedrock_reques """Regression: after Bedrock rejects a replayed thinking block (400 invalid signature), the strip-and-retry re-sign must not inherit attempt 1's SigV4 Authorization/X-Amz-Date; reusing them over the new stripped body makes AWS return 403 SignatureDoesNotMatch.""" - from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeMessagesConfig, ) @@ -5654,3 +5654,44 @@ def test_vector_store_retrieve_handler_sends_id_before_azure_api_version_query() sent: Final = received[0] assert str(sent.url) == "https://x.openai.azure.com/openai/vector_stores/vs_abc?api-version=2025-04-01-preview" assert len(received) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.parametrize("provider", ("openai", "azure")) +async def test_non_chatgpt_responses_preserves_caller_headers_after_validation( + is_async: bool, provider: str, respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=r"https://responses\.example\.com/.*").respond( + 200, + json={ + "id": "resp_headers", "object": "response", "created_at": 1, + "model": "gpt-6-luna", "status": "completed", "output": [], + "parallel_tool_calls": False, "tool_choice": "auto", "tools": [], + }, + ) + client: Final = AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()) if is_async else HTTPHandler(client=httpx.Client()) + kwargs: Final = { + "model": f"{provider}/gpt-6-luna", + "input": "hello", + "api_key": "fake-configured-key", + "api_base": "https://responses.example.com/v1", + "api_version": "2025-04-01-preview", + "extra_headers": {"Authorization": "Bearer fake-caller-key", "content-type": "application/vnd.example+json"}, + "client": client, + } + try: + response: Final = await litellm.aresponses(**kwargs) if is_async else litellm.responses(**kwargs) + assert response.status == "completed" + assert route.calls[0].request.headers["authorization"] == "Bearer fake-caller-key" + expected_content_types: Final = ( + ["application/json", "application/vnd.example+json"] + if provider == "openai" + else ["application/vnd.example+json"] + ) + assert route.calls[0].request.headers.get_list("content-type") == expected_content_types + finally: + if isinstance(client, AsyncHTTPHandler): + await client.close() + else: + client.client.close() diff --git a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py index 86252a1a95d..f55644eccae 100644 --- a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py +++ b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py @@ -314,4 +314,4 @@ class TestElevenLabsTextToSpeechTransformation: ) assert voice_id in url - assert "output_format=pcm_44100" in url + assert "output_format=pcm_24000" in url diff --git a/tests/unit/llms/elevenlabs/test_elevenlabs_text_to_speech_transformation.py b/tests/unit/llms/elevenlabs/test_elevenlabs_text_to_speech_transformation.py index d05371d7df9..88f0ae1d524 100644 --- a/tests/unit/llms/elevenlabs/test_elevenlabs_text_to_speech_transformation.py +++ b/tests/unit/llms/elevenlabs/test_elevenlabs_text_to_speech_transformation.py @@ -55,3 +55,31 @@ def test_speech_keeps_an_internal_prefixed_kwarg_out_of_the_elevenlabs_request(r sent: Final = json.loads(respx_mock.calls[0].request.content) assert "_litellm_undeclared_sentinel" not in sent, sent assert sent["text"] == "hi" + + +@pytest.mark.parametrize( + ("response_format", "expected_output_format"), + [("wav", "wav_24000"), ("pcm", "pcm_24000")], +) +def test_speech_maps_wav_and_pcm_to_elevenlabs_output_formats( + respx_mock: respx.MockRouter, + response_format: str, + expected_output_format: str, +) -> None: + api_base: Final = "http://localhost:12346" + mock_route: Final = respx_mock.post(url__regex=rf"{api_base}/v1/text-to-speech/.*").mock( + return_value=httpx.Response(status_code=200, content=b"audio", headers={"content-type": "audio/mpeg"}) + ) + + litellm.speech( + model="elevenlabs/eleven_multilingual_v2", + input="hi", + voice="21m00Tcm4TlvDq8ikWAM", + response_format=response_format, + api_base=api_base, + api_key="fake_elevenlabs_api_key", + ) + + assert mock_route.called + request_url: Final = httpx.URL(str(respx_mock.calls[0].request.url)) + assert request_url.params["output_format"] == expected_output_format diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 76faf338307..3e96995621e 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -12,7 +12,7 @@ from httpx import Response import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY from litellm.litellm_core_utils.prompt_templates.image_handling import in_memory_cache -from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, diff --git a/tests/unit/llms/openai/test_gpt5_transformation.py b/tests/unit/llms/openai/test_gpt_reasoning_transformation.py similarity index 99% rename from tests/unit/llms/openai/test_gpt5_transformation.py rename to tests/unit/llms/openai/test_gpt_reasoning_transformation.py index 7b10411ba20..4cdc674c774 100644 --- a/tests/unit/llms/openai/test_gpt5_transformation.py +++ b/tests/unit/llms/openai/test_gpt_reasoning_transformation.py @@ -7,7 +7,7 @@ import pytest import litellm import litellm.main as litellm_main from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map -from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config from litellm.llms.openai.openai import OpenAIConfig from litellm.utils import ( is_explicitly_disabled_factory, diff --git a/tests/unit/llms/openai/test_is_model_gpt_5_model.py b/tests/unit/llms/openai/test_is_model_gpt_5_model.py index f6fef92fc6a..39d7c9da2ce 100644 --- a/tests/unit/llms/openai/test_is_model_gpt_5_model.py +++ b/tests/unit/llms/openai/test_is_model_gpt_5_model.py @@ -35,8 +35,8 @@ from typing import Final import pytest import litellm -from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config -from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config +from litellm.llms.openai.chat.gpt_reasoning_transformation import OpenAIGPT5Config from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig # --------------------------------------------------------------------------- diff --git a/tests/unit/llms/openai_like/test_abliteration_provider.py b/tests/unit/llms/openai_like/test_abliteration_provider.py index 33ad8abf62c..74bb5f89781 100644 --- a/tests/unit/llms/openai_like/test_abliteration_provider.py +++ b/tests/unit/llms/openai_like/test_abliteration_provider.py @@ -2,12 +2,18 @@ Unit tests for the Abliteration OpenAI-like provider. """ +import json import os import sys +from typing import Final -sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) -) +import pytest +import respx + +import litellm +from litellm.utils import trim_messages + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../.."))) from litellm.llms.openai_like.dynamic_config import create_config_class from litellm.llms.openai_like.json_loader import JSONProviderRegistry @@ -48,3 +54,79 @@ def test_abliteration_complete_url_appends_endpoint(): stream=False, ) assert url == f"{ABLITERATION_BASE_URL}/chat/completions" + + +def test_abliteration_chat_forwards_reasoning_and_accounts_for_cache(local_model_cost_map: None) -> None: + with respx.mock() as upstream: + route: Final = upstream.post("https://api.abliteration.ai/v1/chat/completions").respond( + 200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "abliterated-model-large-v2", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "hello"}, "finish_reason": "stop"} + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + "prompt_tokens_details": {"cached_tokens": 80}, + }, + }, + ) + response: Final = litellm.completion( + model="abliteration/abliterated-model-large-v2", + api_key="fake-test-key", + messages=[{"role": "user", "content": "Say hello"}], + reasoning_effort="low", + ) + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert request.headers["authorization"] == "Bearer fake-test-key" + assert body["model"] == "abliterated-model-large-v2" + assert body["reasoning_effort"] == "low" + assert response.choices[0].message.content == "hello" + rates: Final = litellm.model_cost["abliteration/abliterated-model-large-v2"] + expected: Final = ( + 20 * rates["input_cost_per_token"] + + 80 * rates["cache_read_input_token_cost"] + + 5 * rates["output_cost_per_token"] + ) + assert litellm.completion_cost(completion_response=response) == pytest.approx(expected) + + +def test_large_v2_model_group_does_not_offer_disabled_reasoning(local_model_cost_map: None) -> None: + """Large v2 cannot disable reasoning per https://docs.abliteration.ai/capabilities/thinking (2026-10-26).""" + router: Final = litellm.Router( + model_list=[ + { + "model_name": "large-v2", + "litellm_params": { + "model": "abliteration/abliterated-model-large-v2", + "api_key": "fake-test-key", + }, + } + ] + ) + info: Final = router.get_model_group_info(model_group="large-v2") + assert info is not None + assert info.supported_reasoning_efforts + assert "none" not in info.supported_reasoning_efforts + assert "low" in info.supported_reasoning_efforts + + +def test_abliteration_models_trim_to_their_input_budget(local_model_cost_map: None) -> None: + model_names: Final = tuple(model for model in litellm.model_cost if model.startswith("abliteration/")) + assert model_names + messages: Final = [{"role": "user", "content": "hello " * 500}] + budget: Final = 100 + for model in model_names: + trimmed: Final = trim_messages( + messages, + model=model, + trim_ratio=budget / litellm.model_cost[model]["max_input_tokens"], + ) + assert litellm.token_counter(model=model, messages=messages) > budget + assert litellm.token_counter(model=model, messages=trimmed) <= budget diff --git a/tests/unit/llms/openai_like/test_openinfer_provider.py b/tests/unit/llms/openai_like/test_openinfer_provider.py index 7bcc489fcf8..6e6e957df0b 100644 --- a/tests/unit/llms/openai_like/test_openinfer_provider.py +++ b/tests/unit/llms/openai_like/test_openinfer_provider.py @@ -11,9 +11,7 @@ from litellm.proxy.auth.model_checks import get_provider_models _REPO_ROOT: Final = Path(__file__).resolve().parents[4] _PRIMARY_PRICES: Final = json.loads((_REPO_ROOT / "model_prices_and_context_window.json").read_text()) -_BACKUP_PRICES: Final = json.loads( - (_REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json").read_text() -) +_BACKUP_PRICES: Final = json.loads((_REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json").read_text()) _CATALOG_MODELS: Final = tuple( key for key, row in _PRIMARY_PRICES.items() if isinstance(row, dict) and row.get("litellm_provider") == "openinfer" ) @@ -31,7 +29,9 @@ _CHAT_RESPONSE: Final = { "object": "chat.completion", "created": 1_792_000_000, "model": "@oi/Qwen3.5-27B", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello from OpenInfer"}, "finish_reason": "stop"}], + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Hello from OpenInfer"}, "finish_reason": "stop"} + ], "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, } @@ -228,7 +228,6 @@ def test_catalog_row_matches_its_backup_and_fits_its_context(model: str): assert row == _BACKUP_PRICES[model] assert row["mode"] == "chat" - assert row["max_tokens"] == row["max_output_tokens"] assert 0 < row["max_output_tokens"] <= row["max_input_tokens"] assert row["input_cost_per_token"] > 0 assert row["output_cost_per_token"] > 0 diff --git a/tests/unit/llms/openai_like/test_refuseless_provider.py b/tests/unit/llms/openai_like/test_refuseless_provider.py new file mode 100644 index 00000000000..4961f100709 --- /dev/null +++ b/tests/unit/llms/openai_like/test_refuseless_provider.py @@ -0,0 +1,111 @@ +import json +from typing import Final + +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.utils import trim_messages + + +def test_refuseless_resolves_environment_and_explicit_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("REFUSELESS_API_KEY", "fake-environment-key") + assert get_llm_provider(model="refuseless/glm-5.3-abliterated") == ( + "glm-5.3-abliterated", + "refuseless", + "fake-environment-key", + "https://api.refuseless.com/v1", + ) + assert get_llm_provider( + model="refuseless/glm-5.3-abliterated", api_key="fake-explicit-key", api_base="https://gateway.example/v1" + ) == ("glm-5.3-abliterated", "refuseless", "fake-explicit-key", "https://gateway.example/v1") + + +def test_refuseless_chat_forwards_reasoning_and_accounts_for_cache(local_model_cost_map: None) -> None: + with respx.mock() as upstream: + route: Final = upstream.post("https://api.refuseless.com/v1/chat/completions").respond( + 200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "glm-5.3-abliterated", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "hello"}, "finish_reason": "stop"} + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + "prompt_tokens_details": {"cached_tokens": 80}, + }, + }, + ) + response: Final = litellm.completion( + model="refuseless/glm-5.3-abliterated", + api_key="fake-test-key", + messages=[{"role": "user", "content": "Say hello"}], + reasoning_effort="low", + ) + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert request.headers["authorization"] == "Bearer fake-test-key" + assert body["model"] == "glm-5.3-abliterated" + assert body["reasoning_effort"] == "low" + assert response.choices[0].message.content == "hello" + rates: Final = litellm.model_cost["refuseless/glm-5.3-abliterated"] + expected: Final = ( + 20 * rates["input_cost_per_token"] + + 80 * rates["cache_read_input_token_cost"] + + 5 * rates["output_cost_per_token"] + ) + assert litellm.completion_cost(completion_response=response) == pytest.approx(expected) + + +def test_refuseless_uses_native_responses_endpoint() -> None: + with respx.mock() as upstream: + route: Final = upstream.post("https://api.refuseless.com/v1/responses").respond( + 200, + json={ + "id": "resp-test", + "object": "response", + "created_at": 1, + "model": "glm-5.3-abliterated", + "status": "completed", + "output": [ + { + "id": "msg-test", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "hello", "annotations": []}], + } + ], + "usage": {"input_tokens": 4, "output_tokens": 3, "total_tokens": 7}, + }, + ) + response: Final = litellm.responses( + model="refuseless/glm-5.3-abliterated", + api_key="fake-test-key", + input="Say hello", + store=False, + ) + assert route.calls.last.request.headers["authorization"] == "Bearer fake-test-key" + assert json.loads(route.calls.last.request.content)["model"] == "glm-5.3-abliterated" + assert response.output[0].content[0].text == "hello" + + +def test_refuseless_models_trim_to_their_input_budget(local_model_cost_map: None) -> None: + model_names: Final = tuple(model for model in litellm.model_cost if model.startswith("refuseless/")) + assert model_names + messages: Final = [{"role": "user", "content": "hello " * 500}] + budget: Final = 100 + for model in model_names: + trimmed: Final = trim_messages( + messages, + model=model, + trim_ratio=budget / litellm.model_cost[model]["max_input_tokens"], + ) + assert litellm.token_counter(model=model, messages=messages) > budget + assert litellm.token_counter(model=model, messages=trimmed) <= budget diff --git a/tests/unit/llms/pass_through/guardrail_translation/test_handler.py b/tests/unit/llms/pass_through/guardrail_translation/test_handler.py index 0dba0d7b90f..7cbabb95931 100644 --- a/tests/unit/llms/pass_through/guardrail_translation/test_handler.py +++ b/tests/unit/llms/pass_through/guardrail_translation/test_handler.py @@ -19,6 +19,22 @@ def test_full_payload_guardrail_text_excludes_the_server_streaming_marker(): assert json.loads(text) == body, text +def test_full_payload_guardrail_text_excludes_the_injected_litellm_metadata(): + body: Final = {"model": "m", "messages": [{"role": "user", "content": "hi"}], "guardrails": ["g"]} + litellm_metadata: Final = { + "user_api_key_token": "hashed-key", + "user_api_key_user_email": "someone@example.com", + "user_api_key_user_role": "proxy_admin", + } + + text: Final = PassThroughEndpointHandler()._extract_text_for_guardrail( + {**body, "litellm_metadata": litellm_metadata}, + None, + ) + + assert json.loads(text) == body, text + + @pytest.mark.parametrize( "marker", [ diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index b97724319c7..75449eacfd3 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -2,7 +2,7 @@ import base64 from collections.abc import Sequence from itertools import chain from pathlib import Path -from typing import Final +from typing import Final, Literal import httpx import pytest @@ -1650,10 +1650,28 @@ def test_convert_tool_response_with_base64_image(): assert inline_data["data"] == test_image_base64 -def test_gemini_history_nests_multimodal_tool_response_parts(): - """Full history conversion should not emit sibling inline_data tool result parts.""" - test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" - messages = [ +@pytest.mark.parametrize("provider", ["gemini", "vertex_ai", "vertex_ai_beta"]) +@pytest.mark.parametrize( + "model,nested", + [ + (None, True), + ("gemini-2.5-pro", False), + ("gemini/gemini-2.5-flash", False), + ("vertex_ai/gemini-2.5-flash-lite", False), + ("123456789", False), + ("gemini-3.8-flash", True), + ("gemini/gemini-3.8-flash", True), + ("vertex_ai/gemini-3.8-flash", True), + ("gemini-flash-latest", True), + ], +) +def test_gemini_history_places_tool_media_for_model( + model: str | None, + nested: bool, + provider: Literal["gemini", "vertex_ai", "vertex_ai_beta"], +) -> None: + test_image_base64: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + messages: Final = [ {"role": "user", "content": "Get me an image"}, { "role": "assistant", @@ -1679,24 +1697,53 @@ def test_gemini_history_nests_multimodal_tool_response_parts(): "data": test_image_base64, }, }, + { + "type": "file", + "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQ="}, + }, ], }, + {"role": "user", "content": "Describe the image and file again"}, ] - contents = gemini_convert_messages_with_history(messages=messages) - - tool_response_parts = contents[-1]["parts"] - assert len(tool_response_parts) == 1 - assert "inline_data" not in tool_response_parts[0] - function_response = tool_response_parts[0]["function_response"] - assert function_response["parts"] == [ + contents: Final = ( + gemini_convert_messages_with_history(messages=messages) + if model is None + else transform_request_body( + messages=messages, + model=model, + custom_llm_provider=provider, + optional_params={}, + litellm_params={}, + cached_content=None, + )["contents"] + ) + media_parts: Final = [ { "inline_data": { "data": test_image_base64, "mime_type": "image/png", } - } + }, + {"inline_data": {"data": "JVBERi0xLjQ=", "mime_type": "application/pdf"}}, ] + function_response: Final = { + "name": "get_image", + "response": {"image_ref": "inline"}, + **({"id": "call_get_image"} if model is not None and nested else {}), + **({"parts": media_parts} if nested else {}), + } + assert contents[-2] == { + "role": "user", + "parts": [ + {"function_response": function_response}, + *([] if nested else media_parts), + ], + } + assert contents[-1] == { + "role": "user", + "parts": [{"text": "Describe the image and file again"}], + } def test_convert_tool_response_text_only(): diff --git a/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py b/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py index 630b2e1eb34..8abd012f9d8 100644 --- a/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py +++ b/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py @@ -5,13 +5,19 @@ Based on the test patterns from other rerank providers and the current Vertex AI import json import os +from datetime import datetime, timedelta +from typing import Final from unittest.mock import MagicMock, patch import httpx import pytest +import responses +import respx +import litellm from litellm.llms.vertex_ai.rerank.transformation import VertexAIRerankConfig from litellm.types.rerank import RerankResponse +from litellm.utils import ProviderConfigManager class TestVertexAIRerankTransform: @@ -646,3 +652,83 @@ class TestVertexAIRerankTransform: second_call = mock_ensure_access_token.call_args_list[1] assert first_call.kwargs["credentials"] == "path/to/credentials.json" assert second_call.kwargs["credentials"] == "path/to/credentials.json" + + +@pytest.fixture +def clear_vertex_rerank_cache(request: pytest.FixtureRequest) -> None: + ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear() + request.addfinalizer( + ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("clear_vertex_rerank_cache") +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +async def test_rerank_reuses_scoped_credentials_and_refreshes_expired_tokens( + is_async: bool, +) -> None: + now: Final = datetime(2025, 1, 1) + credentials: Final = tuple( + json.dumps( + { + "type": "authorized_user", + "client_id": identity, + "client_secret": "synthetic-client-secret", + "refresh_token": f"synthetic-refresh-{identity}", + } + ) + for identity in ("first-principal", "second-principal") + ) + with ( + patch("litellm.disable_aiohttp_transport", True), + patch("google.auth._helpers.utcnow", return_value=now) as clock, + responses.RequestsMock(assert_all_requests_are_fired=False) as token_http, + respx.mock as rank_http, + ): + for token in ("initial", "other-credential", "other-project", "renewed"): + token_http.post( + "https://oauth2.googleapis.com/token", + json={"access_token": token, "expires_in": 3600, "token_type": "Bearer"}, + ) + rank_http.post( + url__regex=r"https://discoveryengine\.googleapis\.com/v1/projects/[^/]+/" + r"locations/global/rankingConfigs/default_ranking_config:rank" + ).respond(200, json={"records": [{"id": "0", "score": 0.9}]}) + + async def rank( + credential: str, project: str, expected_token: str, refresh_count: int + ) -> None: + params: Final = { + "model": "vertex_ai/semantic-ranker-default@latest", + "query": f"Query for {expected_token}", + "documents": ["A relevant document"], + "vertex_credentials": credential, + "vertex_project": project, + "caching": False, + "num_retries": 0, + } + result: Final = ( + await litellm.arerank(**params) + if is_async + else litellm.rerank(**params) + ) + assert isinstance(result, RerankResponse) + assert result.results[0]["index"] == 0 + assert len(token_http.calls) == refresh_count + request: Final = rank_http.calls.last.request + assert request.headers["Authorization"] == f"Bearer {expected_token}" + assert request.headers["X-Goog-User-Project"] == project + assert str(request.url) == ( + f"https://discoveryengine.googleapis.com/v1/projects/{project}/" + "locations/global/rankingConfigs/default_ranking_config:rank" + ) + assert json.loads(request.content)["query"] == params["query"] + + await rank(credentials[0], "first-project", "initial", 1) + await rank(credentials[0], "first-project", "initial", 1) + await rank(credentials[1], "first-project", "other-credential", 2) + await rank(credentials[0], "second-project", "other-project", 3) + await rank(credentials[0], "first-project", "initial", 3) + clock.return_value = now + timedelta(hours=2) + await rank(credentials[0], "first-project", "renewed", 4) diff --git a/tests/unit/llms/vertex_ai/test_vertex_llm_base.py b/tests/unit/llms/vertex_ai/test_vertex_llm_base.py index dd2b86b6f81..fa332cef5d8 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_llm_base.py +++ b/tests/unit/llms/vertex_ai/test_vertex_llm_base.py @@ -6,13 +6,17 @@ import os import pathlib import re import traceback -from typing import Final +from datetime import datetime, timedelta +from threading import Event +from typing import Final, Literal from unittest.mock import MagicMock, call, patch import google.auth.exceptions import pytest +import requests +import responses -from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE, DEFAULT_MAX_RECURSE_DEPTH import litellm @@ -2589,3 +2593,120 @@ def test_get_access_token_reraises_malformed_service_account_credentials_without assert [record.getMessage().split(". Error: ")[0] for record in caplog.records] == [ "Failed to load vertex credentials. Check to see if credentials containing partial/invalid information" ] + + +def _authorized_user_cache_credentials(identity: str) -> str: + return json.dumps( + { + "type": "authorized_user", + "client_id": identity, + "client_secret": "synthetic-secret", + "refresh_token": identity, + "quota_project_id": identity, + } + ) + + +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("inferred_project", [False, True], ids=["explicit", "inferred"]) +@pytest.mark.asyncio +async def test_credential_cache_bounds_retention_and_reloads_least_recently_used( + is_async: bool, inferred_project: bool +) -> None: + vertex: Final = VertexBase() + limit: Final = max(0, DEFAULT_MAX_LRU_CACHE_SIZE) + with ( + patch("google.auth._helpers.utcnow", return_value=datetime(2025, 1, 1)), + responses.RequestsMock() as oauth, + ): + def token_response(request: requests.PreparedRequest) -> tuple[int, dict[str, str], str]: + return 200, {}, json.dumps({"access_token": f"token-{len(oauth.calls) + 1}", "expires_in": 3600}) + + oauth.add_callback(responses.POST, "https://oauth2.googleapis.com/token", callback=token_response) + + async def access(index: int, infer: bool = False) -> tuple[str, str]: + identity: Final = f"project-{index}" + credentials: Final = _authorized_user_cache_credentials(identity) + project: Final = None if infer else identity + result: Final = ( + await vertex.get_access_token_async(credentials, project) + if is_async + else vertex.get_access_token(credentials, project) + ) + assert result[1] == identity + assert len(vertex._credentials_project_mapping) <= limit + return result + + if limit == 0: + assert await access(0, inferred_project) == ("token-1", "project-0") + assert await access(0, inferred_project) == ("token-2", "project-0") + return + + for index in range(limit): + assert await access(index) == (f"token-{index + 1}", f"project-{index}") + assert await access(0) == ("token-1", "project-0") + assert len(oauth.calls) == limit + assert await access(limit, inferred_project) == (f"token-{limit + 1}", f"project-{limit}") + assert len(oauth.calls) == limit + 1 + if limit > (2 if inferred_project else 1): + assert await access(0) == ("token-1", "project-0") + assert len(oauth.calls) == limit + 1 + evicted: Final = 1 if limit > 1 else 0 + assert await access(evicted) == (f"token-{limit + 2}", f"project-{evicted}") + assert len(oauth.calls) == limit + 2 + + +@pytest.mark.parametrize("mode", ["sync", "async", "background"]) +@pytest.mark.skipif(DEFAULT_MAX_LRU_CACHE_SIZE < 2, reason="Eviction during refresh requires two retained entries") +@pytest.mark.asyncio +async def test_evicted_credentials_finish_refresh_without_being_readmitted( + mode: Literal["sync", "async", "background"], +) -> None: + vertex: Final = VertexBase() + limit: Final = DEFAULT_MAX_LRU_CACHE_SIZE + now: Final = datetime(2025, 1, 1) + refresh_started: Final = Event() + release_refresh: Final = Event() + donor: Final = _authorized_user_cache_credentials("donor") + target: Final = _authorized_user_cache_credentials("target") + with patch("google.auth._helpers.utcnow", return_value=now) as clock, responses.RequestsMock() as oauth: + def token_response(request: requests.PreparedRequest) -> tuple[int, dict[str, str], str]: + body: Final = request.body.decode() if isinstance(request.body, bytes) else request.body + assert isinstance(body, str) + is_target: Final = "refresh_token=target" in body + if is_target and len(oauth.calls) == limit: + refresh_started.set() + assert release_refresh.wait(5), "OAuth refresh was not released" + return 200, {}, json.dumps( + {"access_token": f"token-{len(oauth.calls) + 1}", "expires_in": 3600 if is_target else 86400} + ) + + oauth.add_callback(responses.POST, "https://oauth2.googleapis.com/token", callback=token_response) + assert vertex.get_access_token(donor, None) == ("token-1", "donor") + assert vertex.get_access_token(donor, None) == ("token-1", "donor") + for index in range(limit - 2): + assert vertex.get_access_token(donor, f"filler-{index}") == (f"token-{index + 2}", f"filler-{index}") + assert vertex.get_access_token(target, "target") == (f"token-{limit}", "target") + clock.return_value = now + timedelta(seconds=3550 if mode == "background" else 7200) + refresh: Final = asyncio.create_task( + asyncio.to_thread(vertex.get_access_token, target, "target") + if mode == "sync" + else vertex.get_access_token_async(target, "target") + ) + try: + assert await asyncio.to_thread(refresh_started.wait, 5), "OAuth refresh did not start" + for index in range(limit - 2): + assert vertex.get_access_token(donor, f"filler-{index}") == (f"token-{index + 2}", f"filler-{index}") + assert vertex.get_access_token(donor, None) == ("token-1", "donor") + assert len(vertex._credentials_project_mapping) == limit + assert (target, "target") not in vertex._credentials_project_mapping + finally: + release_refresh.set() + completed: Final = await refresh + await asyncio.gather(*tuple(vertex._background_refresh_tasks.values())) + expected_token: Final = f"token-{limit if mode == 'background' else limit + 1}" + assert completed == (expected_token, "target") + assert len(oauth.calls) == limit + 1 + assert (target, "target") not in vertex._credentials_project_mapping + assert vertex.get_access_token(target, "target") == (f"token-{limit + 2}", "target") + assert len(vertex._credentials_project_mapping) == limit diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/llama3/__init__.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/llama/__init__.py similarity index 100% rename from tests/unit/llms/vertex_ai/vertex_ai_partner_models/llama3/__init__.py rename to tests/unit/llms/vertex_ai/vertex_ai_partner_models/llama/__init__.py diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/llama3/test_vertex_ai_partner_models_llama3_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/llama/test_vertex_ai_partner_models_llama_transformation.py similarity index 98% rename from tests/unit/llms/vertex_ai/vertex_ai_partner_models/llama3/test_vertex_ai_partner_models_llama3_transformation.py rename to tests/unit/llms/vertex_ai/vertex_ai_partner_models/llama/test_vertex_ai_partner_models_llama_transformation.py index 05e4e36edd7..72efd7b66b0 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/llama3/test_vertex_ai_partner_models_llama3_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/llama/test_vertex_ai_partner_models_llama_transformation.py @@ -5,7 +5,7 @@ import httpx import pytest -from litellm.llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import ( +from litellm.llms.vertex_ai.vertex_ai_partner_models.llama.transformation import ( VertexAILlama3Config, VertexAILlama3StreamingHandler, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index e8689b2ba89..16130364364 100644 --- a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -7693,6 +7693,47 @@ class TestMCPDcrBridgeDualCredential: assert "invalid_token" in str(exc_info.value.headers) +@pytest.mark.parametrize("public_path", ["/toolset/analysis/mcp", "/analysis/mcp", "/mcp", "/mcp/", "/mcp/proxy"]) +@pytest.mark.parametrize("root_path", ["", "/litellm"]) +@pytest.mark.parametrize("invalid_token", [False, True]) +def test_gateway_challenge_metadata_contains_the_public_resource( + public_path: str, root_path: str, invalid_token: bool, +) -> None: + from mcp.shared.auth_utils import check_resource_allowed + from starlette.requests import Request + from starlette.types import Scope + + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import _gateway_dcr_challenge + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + _build_aggregate_protected_resource_response, + oauth_protected_resource_root, + ) + + scope: Final[Scope] = { + "type": "http", "method": "POST", "scheme": "https", "path": "/mcp", + "root_path": root_path, "_original_path": f"{root_path}{public_path}", + "headers": [(b"host", b"gateway.example.com")], + } + request: Final = Request(scope) + challenge: Final = _gateway_dcr_challenge(request, "/mcp", None, invalid_token) + header: Final = (challenge.headers or {})["WWW-Authenticate"] + metadata_url: Final = header.split('resource_metadata="', 1)[1].split('"', 1)[0] + toolset: Final = public_path in ("/toolset/analysis/mcp", "/analysis/mcp") + assert metadata_url == ( + f"https://gateway.example.com{root_path}/.well-known/oauth-protected-resource" + + ("" if toolset else "/mcp") + ) + metadata: Final = ( + oauth_protected_resource_root(request) if toolset else _build_aggregate_protected_resource_response(request) + ) + assert check_resource_allowed( + requested_resource=f"https://gateway.example.com{root_path}{public_path}", + configured_resource=metadata["resource"], + ) + assert challenge.status_code == 401 + assert ('error="invalid_token"' in header) is invalid_token + + @pytest.mark.asyncio class TestAggregateGatewayDcrChallenge: """The mcp_gateway_dcr front door: a 401 on the aggregate /mcp scope must diff --git a/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py b/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py index f6902c778e5..fdae575276d 100644 --- a/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py +++ b/tests/unit/proxy/_experimental/mcp_server/faults/test_list_outcomes.py @@ -9,9 +9,10 @@ if sys.version_info < (3, 11): # BaseExceptionGroup is a builtin only from 3.11 from exceptiongroup import BaseExceptionGroup import httpx +import httpx2 import pytest from mcp import MCPError -from mcp.types import ErrorData +from mcp.types import CONNECTION_CLOSED, ErrorData from litellm.proxy._experimental.mcp_server.exceptions import ( MCPServerListError, @@ -102,6 +103,68 @@ def test_unknown_exception_is_internal(): assert classify_list_exception(ValueError("who knows")).tag == "internal" +@pytest.mark.parametrize("sdk_wrapped", (False, True)) +@pytest.mark.parametrize("nested", (False, True)) +def test_sse_event_limit_is_a_safe_upstream_fault(sdk_wrapped: bool, nested: bool) -> None: + limit: Final = 1024 + decoder_message: Final = f"Server-sent event exceeded the {limit} byte limit." + failure: Final = ( + MCPError(code=CONNECTION_CLOSED, message=f"SSE stream failed: {decoder_message}") + if sdk_wrapped else httpx2.SSEError(decoder_message) + ) + wrapped: Final = BaseExceptionGroup("private-upstream-secret", [failure]) if nested else failure + fault: Final = classify_list_exception(wrapped) + message: Final = ( + f"Upstream MCP SSE event exceeded the {limit} byte per-event limit. " + "Paginate tools/list responses so each event stays below the limit." + ) + assert fault == ServerListFault(tag="upstream_error", sse_event_limit_bytes=limit) + assert fault.public_message == message + assert list_fault_http_status(fault) == 502 + assert outcome_wire_value(fault) == { + "status": "upstream_error", "reason": "sse_event_too_large", "limit_bytes": limit, "message": message, + } + + +@pytest.mark.parametrize( + "failure", + ( + MCPError(code=CONNECTION_CLOSED, message="Connection closed"), + MCPError(code=-32603, message="SSE stream failed: Server-sent event exceeded the 1024 byte limit."), + MCPError(code=CONNECTION_CLOSED, message="SSE stream failed: Server-sent event exceeded the 0 byte limit."), + MCPError(code=CONNECTION_CLOSED, message="SSE stream failed: Server-sent event exceeded the -1 byte limit."), + MCPError(code=CONNECTION_CLOSED, message=( + "SSE stream failed: Server-sent event exceeded the 10000000000000000000 byte limit." + )), + MCPError(code=CONNECTION_CLOSED, message=( + "SSE stream failed: Server-sent event exceeded the 1024 byte limit. private-secret" + )), + httpx2.SSEError("private-secret Server-sent event exceeded the 1024 byte limit."), + httpx2.SSEError("Expected response with content type 'text/event-stream', got 'private-secret'."), + RuntimeError("SSE stream failed: Server-sent event exceeded the 1024 byte limit."), + ), +) +def test_unrelated_errors_do_not_claim_an_sse_event_limit(failure: Exception) -> None: + fault: Final = classify_list_exception(failure) + assert fault.sse_event_limit_bytes is None + assert fault.public_message is None + assert "private-secret" not in str(outcome_wire_value(fault)) + + +@pytest.mark.parametrize("status_code", (401, 403)) +def test_upstream_auth_takes_precedence_over_sse_event_limit(status_code: int) -> None: + request: Final = httpx2.Request("POST", "https://upstream.example/mcp") + response: Final = httpx2.Response(status_code, request=request) + failure: Final = BaseExceptionGroup("upstream errors", [ + httpx2.SSEError("Server-sent event exceeded the 1024 byte limit."), + httpx2.HTTPStatusError("private-secret", request=request, response=response), + ]) + fault: Final = classify_list_exception(failure) + assert fault.tag == ("auth_required" if status_code == 401 else "forbidden") + assert list_fault_http_status(fault) == status_code + assert fault.sse_event_limit_bytes is None + + def test_wire_value_carries_no_prose(): fault = ServerListFault(tag="upstream_error", status_code=500) assert outcome_wire_value(fault) == {"status": "upstream_error", "http_status": 500} diff --git a/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index a43dacaf100..cedfa296515 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -5599,12 +5599,13 @@ def _bridge_server(**overrides): return MCPServer(**fields) -def _bridge_mock_request(): +def _bridge_mock_request(cookie: str | None = None) -> MagicMock: from fastapi import Request - mock_request = MagicMock(spec=Request) + mock_request: Final = MagicMock(spec=Request) mock_request.base_url = "https://litellm.example.com/" mock_request.headers = {} + mock_request.cookies = {"token": cookie} if cookie else {} return mock_request @@ -6524,7 +6525,7 @@ async def test_interactive_bridge_gateway_code_for_another_server_is_rejected_40 @pytest.mark.asyncio -async def test_interactive_bridge_authorize_seals_sso_user_into_state(): +async def test_interactive_bridge_authorize_seals_sso_user_into_state(monkeypatch: pytest.MonkeyPatch) -> None: """On the short-circuit bridge oauth_delegate arm, authorize captures the SSO user from the UI session cookie and seals it (and the target server) into the encrypted OAuth state, so the callback can later mint a user-bound gateway code; it still proceeds to the upstream redirect. @@ -6540,16 +6541,13 @@ async def test_interactive_bridge_authorize_seals_sso_user_into_state(): admitted = UserAPIKeyAuth(user_id="sso-user-42") admitted.mcp_admitted_user_subject = True captured: dict = {} + cookie: Final = _browser_session_cookie(monkeypatch, "sso-user-42") def _capture(**kwargs): captured.update(kwargs) return "mocked_encrypted_state" with ( - patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.user_id_from_session_cookie", - return_value="sso-user-42", - ), patch( "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", new=AsyncMock(return_value=admitted), @@ -6565,7 +6563,7 @@ async def test_interactive_bridge_authorize_seals_sso_user_into_state(): ), ): response = await authorize_with_server( - request=_bridge_mock_request(), + request=_bridge_mock_request(cookie), mcp_server=server, client_id="ignored", redirect_uri="http://127.0.0.1:60108/callback", @@ -6581,7 +6579,9 @@ async def test_interactive_bridge_authorize_seals_sso_user_into_state(): @pytest.mark.asyncio @pytest.mark.parametrize("user_can_reach_server", [True, False]) -async def test_bridge_authorize_gates_on_the_egress_server_access_resolver(user_can_reach_server): +async def test_bridge_authorize_gates_on_the_egress_server_access_resolver( + monkeypatch: pytest.MonkeyPatch, user_can_reach_server: bool +) -> None: """The interactive dcr_bridge oauth_delegate authorize admits the signed-in user the way MCP egress will and refuses with an RFC 6749 access_denied redirect when that admitted subject cannot reach the target server, instead of minting an envelope whose every tool request would @@ -6607,14 +6607,11 @@ async def test_bridge_authorize_gates_on_the_egress_server_access_resolver(user_ mock_request = MagicMock(spec=Request) mock_request.base_url = "https://litellm.example.com/" mock_request.headers = {} + mock_request.cookies = {"token": _browser_session_cookie(monkeypatch, "bridge-user-1")} client_redirect = "http://127.0.0.1:60108/callback" try: with ( - patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.user_id_from_session_cookie", - return_value="bridge-user-1", - ), patch( "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", new=AsyncMock(return_value=admitted), @@ -6660,7 +6657,9 @@ async def test_bridge_authorize_gates_on_the_egress_server_access_resolver(user_ @pytest.mark.asyncio @pytest.mark.parametrize("reload_status,expect_denial", [(401, True), (500, False), (503, False)]) -async def test_bridge_authorize_reload_failure_denies_or_stays_retryable(reload_status, expect_denial): +async def test_bridge_authorize_reload_failure_denies_or_stays_retryable( + monkeypatch: pytest.MonkeyPatch, reload_status: int, expect_denial: bool +) -> None: """An unknown or deactivated signed-in user denies like a missing grant (fail closed); a DB outage keeps its retryable 503 instead of masquerading as an access denial.""" from urllib.parse import parse_qs, urlparse @@ -6679,13 +6678,10 @@ async def test_bridge_authorize_reload_failure_denies_or_stays_retryable(reload_ mock_request = MagicMock(spec=Request) mock_request.base_url = "https://litellm.example.com/" mock_request.headers = {} + mock_request.cookies = {"token": _browser_session_cookie(monkeypatch, "bridge-user-1")} try: with ( - patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.user_id_from_session_cookie", - return_value="bridge-user-1", - ), patch( "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler.reload_admitted_user", new=AsyncMock(side_effect=HTTPException(status_code=reload_status, detail="x")), @@ -6728,19 +6724,15 @@ async def test_interactive_bridge_authorize_without_session_redirects_to_login() from litellm.types.mcp import MCPAuth server = _bridge_server(auth_type=MCPAuth.oauth_delegate, client_id="admin-client", registration_url=None) - with patch( - "litellm.proxy._experimental.mcp_server.byok_oauth_endpoints.user_id_from_session_cookie", - return_value=None, - ): - response = await authorize_with_server( - request=_bridge_mock_request(), - mcp_server=server, - client_id="ignored", - redirect_uri="http://127.0.0.1:60108/callback", - state="s", - code_challenge="chal", - code_challenge_method="S256", - ) + response = await authorize_with_server( + request=_bridge_mock_request(), + mcp_server=server, + client_id="ignored", + redirect_uri="http://127.0.0.1:60108/callback", + state="s", + code_challenge="chal", + code_challenge_method="S256", + ) assert "/sso/key/generate" in response.headers["location"] @@ -11997,10 +11989,97 @@ def _s256(verifier: str) -> str: _NATIVE_CLIENT_MASTER_KEY = "sk-test-salt-for-LIT-5874" +def _browser_session_cookie( + monkeypatch: pytest.MonkeyPatch, + user_id: str, + *, + signing_key: str = _NATIVE_CLIENT_MASTER_KEY, + session_key: str | None = None, + expired_session: bool = False, + expired_cookie: bool = False, +) -> str: + import jwt + + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import LITELLM_SESSION_TOKEN_PREFIX + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_bearer_token + + monkeypatch.setattr(proxy_server, "master_key", signing_key) + if proxy_server.prisma_client is None: + monkeypatch.setattr(proxy_server, "prisma_client", object()) + monkeypatch.setattr(proxy_server, "general_settings", {**proxy_server.general_settings, "supported_db_objects": []}) + session: Final = UserAPIKeyAuth( + user_id=user_id, + expires=datetime(2000 if expired_session else 2099, 1, 1, tzinfo=timezone.utc), + ) + key: Final = ( + encrypt_bearer_token(session.model_dump_json(exclude_none=True), LITELLM_SESSION_TOKEN_PREFIX) + if session_key is None + else session_key + ) + return jwt.encode( + {"user_id": user_id, "key": key, "login_method": "sso", "exp": 946684800 if expired_cookie else 9999999999}, + signing_key, + algorithm="HS256", + ) + + +class _BrowserSessionDatabase: + def __init__(self) -> None: + from litellm.proxy._types import UserAPIKeyAuth, hash_token + + self.row: UserAPIKeyAuth | None = UserAPIKeyAuth( + token=hash_token("sk-browser-session"), + user_id="u1", + expires=datetime(2099, 1, 1, tzinfo=timezone.utc), + ) + self.litellm_deprecatedverificationtoken = self + + def __call__(self, **kwargs: object) -> "_BrowserSessionDatabase": + return self + + async def query_first(self, query: str, *args: object) -> dict[str, object] | None: + if self.row is None: + return None + return { + **self.row.model_dump(exclude={"last_refreshed_at"}), + "team_models": None, + "team_blocked": None, + "team_members_with_roles": None, + } + + async def find_first(self, **kwargs: object) -> None: + return None + + +@pytest.fixture +def browser_session_database(monkeypatch: pytest.MonkeyPatch) -> _BrowserSessionDatabase: + import prisma + + from litellm.proxy import proxy_server + from litellm.proxy._types import hash_token + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.utils import PrismaClient + + database: Final = _BrowserSessionDatabase() + monkeypatch.setattr(prisma, "Prisma", database) + for name in ("DATABASE_URL_READ_REPLICA", "IAM_TOKEN_DB_AUTH", "AZURE_POSTGRESQL_AUTH"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr( + proxy_server, + "prisma_client", + PrismaClient("postgresql://local-test@localhost/browser-session", proxy_server.proxy_logging_obj), + ) + cache: Final = UserApiKeyCache() + cache.set_cache(hash_token("sk-browser-session"), database.row) + monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) + return database + + def _native_client_app(monkeypatch): """The unauthenticated discoverable router served over TestClient with a signed UI session cookie available, plus fakes for the two database-backed hooks the native-client flow calls.""" - import jwt from fastapi import FastAPI from fastapi.testclient import TestClient @@ -12029,14 +12108,108 @@ def _native_client_app(monkeypatch): app = FastAPI() app.include_router(router) client = TestClient(app) - session_cookie = jwt.encode( - {"user_id": "u1", "login_method": "username_password", "exp": int(time.time()) + 600}, - _NATIVE_CLIENT_MASTER_KEY, - algorithm="HS256", - ) + session_cookie: Final = _browser_session_cookie(monkeypatch, "u1") return client, session_cookie, minted +@pytest.mark.parametrize( + "authorize_path,resource", + [("/authorize", None), ("/authorize/mcp-session", None), ("/authorize", "http://testserver")], +) +@pytest.mark.parametrize( + "session_state", ["valid", "revoked", "blocked", "expired", "wrong_owner", "missing_key", "no_database"] +) +def test_gateway_authorize_requires_live_browser_session( + monkeypatch: pytest.MonkeyPatch, + browser_session_database: _BrowserSessionDatabase, + authorize_path: str, + resource: str | None, + session_state: str, +) -> None: + from urllib.parse import urlparse + + from litellm.proxy import proxy_server + + client, _, _ = _native_client_app(monkeypatch) + database: Final = browser_session_database + assert database.row is not None + database.row = None if session_state == "revoked" else database.row.model_copy( + update={ + "blocked": session_state == "blocked", + "expires": datetime(2000 if session_state == "expired" else 2099, 1, 1, tzinfo=timezone.utc), + "user_id": "another-user" if session_state == "wrong_owner" else "u1", + } + ) + cookie: Final = _browser_session_cookie( + monkeypatch, "u1", session_key="" if session_state == "missing_key" else "sk-browser-session" + ) + client.cookies.set("token", cookie) + if session_state == "no_database": + monkeypatch.setattr(proxy_server, "prisma_client", None) + registered: Final = client.post("/register", json={"redirect_uris": ["http://127.0.0.1/callback"]}) + assert registered.status_code == 201, registered.text + response: Final = client.get( + authorize_path, + params={ + "client_id": registered.json()["client_id"], + "redirect_uri": "http://127.0.0.1/callback", + "response_type": "code", + "code_challenge": _s256("v" * 43), + "code_challenge_method": "S256", + **({"resource": resource} if resource else {}), + }, + follow_redirects=False, + ) + if session_state == "valid" and resource: + assert response.status_code == 200 and 'name="decision"' in response.text, response.text + else: + assert response.status_code == 303, response.text + assert urlparse(response.headers["location"]).path == ( + "/ui/connect" if session_state == "valid" else "/sso/key/generate" + ) + + +@pytest.mark.parametrize("endpoint", ["flow", "complete"]) +@pytest.mark.parametrize("revoked", [False, True]) +def test_gateway_connect_rechecks_session_after_authorize( + monkeypatch: pytest.MonkeyPatch, + browser_session_database: _BrowserSessionDatabase, + endpoint: str, + revoked: bool, +) -> None: + from urllib.parse import parse_qs, urlparse + + client, _, _ = _native_client_app(monkeypatch) + client.cookies.set("token", _browser_session_cookie(monkeypatch, "u1", session_key="sk-browser-session")) + registered: Final = client.post("/register", json={"redirect_uris": ["http://127.0.0.1/callback"]}) + assert registered.status_code == 201, registered.text + authorized: Final = client.get( + "/authorize/mcp-session", + params={ + "client_id": registered.json()["client_id"], + "redirect_uri": "http://127.0.0.1/callback", + "response_type": "code", + "code_challenge": _s256("v" * 43), + "code_challenge_method": "S256", + }, + follow_redirects=False, + ) + flow: Final = parse_qs(urlparse(authorized.headers["location"]).query)["connect_flow"][0] + if revoked: + browser_session_database.row = None + response: Final = ( + client.get("/authorize/flow", params={"flow": flow}) + if endpoint == "flow" + else client.post("/authorize/complete", data={"flow": flow}, follow_redirects=False) + ) + assert response.status_code == (401 if revoked else 200 if endpoint == "flow" else 303), response.text + if revoked: + assert response.json()["error"] == "login_required" + assert "location" not in response.headers + elif endpoint == "complete": + assert parse_qs(urlparse(response.headers["location"]).query)["code"] + + @pytest.mark.parametrize("authorize_path", ["/authorize/mcp-session", "/authorize"]) def test_gateway_authorize_routes_forward_server_scope(monkeypatch: pytest.MonkeyPatch, authorize_path: str) -> None: from http.cookies import SimpleCookie @@ -13605,11 +13778,7 @@ async def test_identity_bound_authorize_preserves_presented_jwt_permissions( else "https://idp.example.test", ) ) - cookie: Final = jwt.encode( - {"user_id": "cookie-owner", "login_method": "sso", "exp": int(time.time()) + 300}, - master, - algorithm="HS256", - ) + cookie: Final = _browser_session_cookie(monkeypatch, "cookie-owner", signing_key=master) response: Final = await discoverable_endpoints.authorize_with_server( request=_token_request( { @@ -13645,7 +13814,7 @@ async def test_identity_bound_authorize_preserves_presented_jwt_permissions( @pytest.mark.asyncio @pytest.mark.parametrize("credential", ["none", "opaque", "foreign_jwt"]) -@pytest.mark.parametrize("cookie_state", ["allowed", "server_denied", "expired", "missing"]) +@pytest.mark.parametrize("cookie_state", ["allowed", "server_denied", "expired", "missing", "session_expired"]) async def test_identity_bound_authorize_unrelated_bearer_uses_browser_session( jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], monkeypatch: pytest.MonkeyPatch, @@ -13654,8 +13823,6 @@ async def test_identity_bound_authorize_unrelated_bearer_uses_browser_session( ) -> None: from urllib.parse import parse_qs, urlparse - import jwt - from litellm.models.user import LiteLLM_UserTable from litellm.proxy import proxy_server from litellm.proxy._experimental.mcp_server import discoverable_endpoints, mcp_server_manager @@ -13694,14 +13861,12 @@ async def test_identity_bound_authorize_unrelated_bearer_uses_browser_session( if credential == "foreign_jwt" else "unrelated-upstream-bearer" ) - cookie: Final = jwt.encode( - { - "user_id": "cookie-owner", - "login_method": "sso", - "exp": int(time.time()) + (-60 if cookie_state == "expired" else 300), - }, - master, - algorithm="HS256", + cookie: Final = _browser_session_cookie( + monkeypatch, + "cookie-owner", + signing_key=master, + expired_cookie=cookie_state == "expired", + expired_session=cookie_state == "session_expired", ) response: Final = await discoverable_endpoints.authorize_with_server( request=_token_request( diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py index b0d53f9c93b..34b021df2b9 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py @@ -5,6 +5,7 @@ from litellm.proxy._experimental.mcp_server import operations as mcp_operations import asyncio import logging import sys +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx @@ -531,12 +532,12 @@ async def test_scope_challenge_replaces_pending_stream_headers_once(): send = AsyncMock() response = OAuthScopeResponse(send, "https://gateway.example", "/docs/mcp") - await response({"type": "http.response.start", "status": 200, "headers": []}) + headers: Final = asyncio.create_task(response({"type": "http.response.start", "status": 200, "headers": []})) body = asyncio.create_task(response({"type": "http.response.body", "body": b"event: message\n"})) await asyncio.sleep(0) send.assert_not_awaited() response.deny(MCPUpstreamAuthError(403, None, "docs", required_scope="tools.write")) - await asyncio.wait_for(body, 1) + await asyncio.wait_for(asyncio.gather(headers, body), 1) await response({"type": "http.response.body", "body": b"private tool error"}) assert send.await_count == 2 start, result = [call.args[0] for call in send.await_args_list] @@ -567,7 +568,7 @@ async def test_scope_authorization_releases_progress_without_waiting_for_tool_re context = ServerRequestContext(session=MagicMock(), lifespan_context={}, protocol_version="2025-11-25", method="tools/call", request=request) token = active_mcp_request_ctx_var.set(context) try: - await gate({"type": "http.response.start", "status": 200, "headers": []}) + headers: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) progress = asyncio.create_task(gate({"type": "http.response.body", "body": b"progress", "more_body": True})) await asyncio.sleep(0) for method, status in [("initialize", 200), ("tools/list", 200), ("tools/call", 403)]: @@ -582,7 +583,7 @@ async def test_scope_authorization_releases_progress_without_waiting_for_tool_re upstream = MCPClient(server_url="https://upstream.example/mcp") async with upstream._create_httpx_client_factory(transport=httpx2.MockTransport(lambda request: httpx2.Response(200, content=b"tool result")))() as client: async with client.stream("POST", "https://upstream.example/mcp", json={"method": "tools/call"}): - await asyncio.wait_for(progress, 1) + await asyncio.wait_for(asyncio.gather(headers, progress), 1) assert send.await_args_list[-1].args[0]["body"] == b"progress" gate.deny(MCPUpstreamAuthError(403, None, "docs", required_scope="tools.write")) await gate({"type": "http.response.body", "body": b"result"}) @@ -594,21 +595,105 @@ async def test_scope_authorization_releases_progress_without_waiting_for_tool_re @pytest.mark.asyncio @pytest.mark.parametrize("failure", [ValueError, asyncio.CancelledError]) -async def test_scope_middleware_releases_waiting_response_on_validation_or_cancellation(failure): +@pytest.mark.parametrize("fail_send", [False, True]) +async def test_scope_middleware_releases_waiting_response_on_validation_or_cancellation( + failure: type[BaseException], fail_send: bool, +) -> None: from mcp.server.context import ServerRequestContext from starlette.requests import Request from litellm.proxy._experimental.mcp_server.scope_challenge import SCOPE_RESPONSE_KEY, OAuthScopeResponse, finish_scope_response - send = AsyncMock() - gate = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") - context = ServerRequestContext(session=MagicMock(), lifespan_context={}, protocol_version="2025-11-25", method="tools/call", request=Request({"type": "http", SCOPE_RESPONSE_KEY: gate})) + send: Final = AsyncMock(side_effect=BrokenPipeError("peer disconnected") if fail_send else None) + gate: Final = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") + context: Final = ServerRequestContext(session=MagicMock(), lifespan_context={}, protocol_version="2025-11-25", method="tools/call", request=Request({"type": "http", SCOPE_RESPONSE_KEY: gate})) + headers: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) + await asyncio.sleep(0) with pytest.raises(failure): await finish_scope_response(context, AsyncMock(side_effect=failure)) - await gate({"type": "http.response.start", "status": 200, "headers": []}) + if fail_send: + with pytest.raises(BrokenPipeError, match="peer disconnected"): + await headers + else: + await headers await gate({"type": "http.response.body", "body": b"error"}) assert not gate.pending - assert send.await_args_list[-1].args[0]["body"] == b"error" + assert send.await_args_list[-1].args[0]["type"] == ("http.response.start" if fail_send else "http.response.body") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fail_send", [False, True]) +async def test_scope_transport_owns_header_send_cancellation_and_errors(fail_send: bool) -> None: + import httpx2 + from starlette.types import Message + + from litellm.proxy._experimental.mcp_server.legacy_callbacks import record_upstream_tool_authorization + from litellm.proxy._experimental.mcp_server.scope_challenge import OAuthScopeResponse + + send_started: Final = asyncio.Event() + send_stopped: Final = asyncio.Event() + writable: Final = asyncio.Event() + sent: Final[list[Message]] = [] + + async def send(message: Message) -> None: + sent.append(message) + assert message["type"] == "http.response.start" + send_started.set() + try: + if fail_send: + raise BrokenPipeError("peer disconnected") + await writable.wait() + finally: + send_stopped.set() + + gate: Final = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") + headers: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) + body: Final = asyncio.create_task(gate({"type": "http.response.body", "body": b"result"})) + await asyncio.sleep(0) + assert not send_started.is_set() + response: Final = httpx2.Response(200, request=httpx2.Request( + "POST", "https://upstream.example/mcp", json={"method": "tools/call"}, + )) + try: + await record_upstream_tool_authorization(response, pending=gate) + await send_started.wait() + if fail_send: + with pytest.raises(BrokenPipeError, match="peer disconnected"): + await headers + else: + headers.cancel() + with pytest.raises(asyncio.CancelledError): + await headers + await body + assert len(sent) == 1 + assert send_stopped.is_set() + finally: + writable.set() + await send_stopped.wait() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("allowed_before_start", [False, True]) +async def test_scope_authorization_sends_stream_headers_before_the_first_body_chunk(allowed_before_start: bool) -> None: + from starlette.types import Message + + from litellm.proxy._experimental.mcp_server.scope_challenge import OAuthScopeResponse + + send: Final = AsyncMock() + gate: Final = OAuthScopeResponse(send, "https://gateway.example", "/mcp/docs") + start: Final[Message] = {"type": "http.response.start", "status": 200, "headers": [(b"content-type", b"text/event-stream")]} + if allowed_before_start: + gate.allow() + headers: Final = asyncio.create_task(gate(start)) + await asyncio.sleep(0) + if not allowed_before_start: + send.assert_not_awaited() + gate.allow() + await headers + send.assert_awaited_once_with(start) + await gate({"type": "http.response.body", "body": b"event: message\n", "more_body": True}) + assert send.await_args_list[-1].args[0]["body"] == b"event: message\n" + assert send.await_count == 2 @pytest.mark.asyncio @@ -697,13 +782,14 @@ async def test_scope_challenges_are_isolated_between_concurrent_requests(): context = ServerRequestContext(session=MagicMock(), lifespan_context={}, protocol_version="2025-11-25", method="tools/call", request=Request({"type": "http", SCOPE_RESPONSE_KEY: gate})) token = active_mcp_request_ctx_var.set(context) try: - await gate({"type": "http.response.start", "status": 200, "headers": []}) + headers: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) await asyncio.sleep(0) assert get_scope_response() is gate if denied: gate.deny(MCPUpstreamAuthError(403, None, "docs", required_scope="tools.write")) else: await record_upstream_tool_authorization(httpx2.Response(200, request=httpx2.Request("POST", "https://upstream.example/mcp", json={"method": "tools/call"})), pending=get_scope_response()) + await headers await gate({"type": "http.response.body", "body": b"done"}) return send.await_args_list[0].args[0]["status"] finally: diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 7315315a619..ecd08e17cda 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -7681,7 +7681,7 @@ class TestMCPServerManager: request=Request({"type": "http", SCOPE_RESPONSE_KEY: gate}), ) token = active_mcp_request_ctx_var.set(context) - await gate({"type": "http.response.start", "status": 200, "headers": []}) + headers: Final = asyncio.create_task(gate({"type": "http.response.start", "status": 200, "headers": []})) progress = asyncio.create_task(gate({"type": "http.response.body", "body": b"progress", "more_body": True})) async def upstream_call(*args, **kwargs): @@ -7701,7 +7701,7 @@ class TestMCPServerManager: assert not result.is_error finally: gate.allow() - await asyncio.wait_for(progress, 1) + await asyncio.wait_for(asyncio.gather(headers, progress), 1) active_mcp_request_ctx_var.reset(token) @pytest.mark.asyncio diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py index b8e28c3f9b8..a873e2ab1ae 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -280,7 +280,7 @@ async def test_mcp_server_tool_call_relays_upstream_auth_error_as_iserror( send = AsyncMock() response = OAuthScopeResponse(send, "https://gateway.example", "/mcp/pt") request = Request({"type": "http", "headers": [], SCOPE_RESPONSE_KEY: response}) - await response({"type": "http.response.start", "status": 200, "headers": []}) + headers: Final = asyncio.create_task(response({"type": "http.response.start", "status": 200, "headers": []})) try: from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError from litellm.proxy._experimental.mcp_server.server import ( @@ -328,6 +328,7 @@ async def test_mcp_server_tool_call_relays_upstream_auth_error_as_iserror( info_calls = [str(c.args[0]) for c in mock_logger.info.call_args_list if c.args] assert any("Upstream auth failure" in m for m in info_calls) response.allow() + await headers await response({"type": "http.response.body", "body": b"tool error"}) start = send.await_args_list[0].args[0] assert start["status"] == (403 if required_scope is not None else 200) diff --git a/tests/unit/proxy/auth/test_auth_exception_handler.py b/tests/unit/proxy/auth/test_auth_exception_handler.py index d83a8aa2fcd..e21491fd996 100644 --- a/tests/unit/proxy/auth/test_auth_exception_handler.py +++ b/tests/unit/proxy/auth/test_auth_exception_handler.py @@ -1,5 +1,7 @@ import asyncio import json +from collections.abc import Mapping +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -29,6 +31,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import INVALID_VIRTUAL_KEY_ERROR_MARKER from litellm.exceptions import BudgetExceededError +from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import ( ModelAccessDeniedProxyException, ProxyErrorTypes, @@ -812,6 +815,65 @@ def _http_request(client_host: str | None = "10.1.2.3", headers: dict[str, str] ) +class _AuthFailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.data: dict[str, object] | None = None + self.error: Exception | None = None + + async def async_post_call_failure_hook( + self, + request_data: dict[str, object], + original_exception: Exception, + user_api_key_dict: UserAPIKeyAuth, + traceback_str: str | None = None, + ) -> HTTPException | None: + self.data = dict(request_data) + self.error = original_exception + return None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_data", [[], None, "metadata", 1, True]) +@pytest.mark.parametrize("error_kind", ["auth", "budget", "database"]) +async def test_auth_error_preserves_classification_with_non_object_context( + monkeypatch: pytest.MonkeyPatch, request_data: object, error_kind: str +) -> None: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(litellm, "budget_exceeded_status_code", 429) + errors: Final = { + "auth": HTTPException(status_code=401, detail="Invalid API key"), + "budget": BudgetExceededError(current_cost=2, max_budget=1), + "database": httpx.ConnectError("Database unavailable"), + } + error: Final = errors[error_kind] + expected: Final = _as_proxy_exception(error) + recorder: Final = _AuthFailureRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + monkeypatch.setattr(proxy_server, "general_settings", {"allow_requests_on_db_unavailable": False}) + + with pytest.raises(ProxyException) as exc_info: + await UserAPIKeyAuthExceptionHandler.handle_authentication_error( + error, + _http_request(headers={"user-agent": "shape-regression"}), + request_data, + "/key/info", + None, + "sk-test", + ) + + assert exc_info.value.code == expected.code + assert exc_info.value.type == expected.type + assert exc_info.value.message == expected.message + assert recorder.error is error + assert recorder.data is not None + metadata: Final = recorder.data["metadata"] + assert isinstance(metadata, Mapping) + assert metadata["requester_ip_address"] == "10.1.2.3" + assert metadata["user_agent"] == "shape-regression" + + @pytest.mark.asyncio @pytest.mark.parametrize( "auth_error, general_settings, request_kwargs, expected_ip", diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index ee1ae43fd3a..10a0f974a42 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -3626,6 +3626,18 @@ class TestIsRequestBodySafeBlocksEndpointTargetingFields: ) +@pytest.mark.parametrize("allow_client_side_credentials", [False, True]) +@pytest.mark.parametrize("nested", [False, True]) +def test_request_cannot_select_chatgpt_auth_file(allow_client_side_credentials: bool, nested: bool) -> None: + selector: Final = {"chatgpt_auth_file": "/tokens/account-b/auth.json"} + with pytest.raises(ValueError, match="chatgpt_auth_file"): + is_request_body_safe( + request_body={"model": "chatgpt-account-a", **({"extra_body": selector} if nested else selector)}, + general_settings={"allow_client_side_credentials": allow_client_side_credentials}, + llm_router=None, + model="chatgpt-account-a", + ) + class TestIsRequestBodySafeBlocksBedrockProjectOverride: """``aws_bedrock_project_id`` pins a deployment to a Bedrock project so that project's data-retention policy applies to its requests. A @@ -5304,3 +5316,27 @@ async def test_a_non_numeric_max_request_size_mb_is_ignored_without_premium(): patch("litellm.proxy.proxy_server.premium_user", False), ): assert await check_if_request_size_is_safe(request=_request_with_content_length(1024)) is True + + +@pytest.mark.parametrize("allow_client_side_credentials", [False, True]) +@pytest.mark.parametrize("route", ["/model/new", "/model/update", "/model/account-a/update", "/health/test_connection"]) +def test_chatgpt_model_configuration_reaches_role_authorization( + allow_client_side_credentials: bool, route: str +) -> None: + body: Final = {"litellm_params": {"model": "chatgpt/gpt-6-luna", "chatgpt_auth_file": "/tokens/account-a/auth.json"}} + assert is_request_body_safe( + request_body=body, + general_settings={"allow_client_side_credentials": allow_client_side_credentials}, + llm_router=None, + model="", + route=route, + ) + for inference_route in ("/v1/chat/completions", "/v1/responses"): + with pytest.raises(ValueError, match="chatgpt_auth_file"): + is_request_body_safe( + request_body=body, + general_settings={"allow_client_side_credentials": allow_client_side_credentials}, + llm_router=None, + model="", + route=inference_route, + ) diff --git a/tests/unit/proxy/auth/test_search_tool_deny_by_default.py b/tests/unit/proxy/auth/test_search_tool_deny_by_default.py index cd285e0386f..da036795889 100644 --- a/tests/unit/proxy/auth/test_search_tool_deny_by_default.py +++ b/tests/unit/proxy/auth/test_search_tool_deny_by_default.py @@ -1,8 +1,9 @@ from collections.abc import Mapping -from typing import Final, TypedDict +from typing import Final, TypedDict, cast from unittest.mock import AsyncMock, MagicMock import pytest +from fastapi import HTTPException from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS from litellm.proxy import proxy_server @@ -78,6 +79,14 @@ def _team_loader(team_tools: list[str] | None | str) -> TeamObjectLoader: return load +def _missing_dashboard_team_loader() -> tuple[TeamObjectLoader, AsyncMock]: + missing_team: Final = HTTPException( + status_code=404, detail="Team doesn't exist in db. Team=litellm-dashboard" + ) + load_team: Final = AsyncMock(side_effect=missing_team) + return cast(TeamObjectLoader, load_team), load_team + + async def _no_team() -> None: return None @@ -194,9 +203,6 @@ async def test_search_tool_access_is_unchanged_without_search_tool_deny_by_defau id="proxy admin virtual key not exempt", ), pytest.param(CallerFields(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS), None, None, None, id="master key exempt"), - pytest.param( - CallerFields(team_id=UI_TEAM_ID, virtual_key=False), None, None, None, id="dashboard session exempt" - ), ], ) @pytest.mark.asyncio @@ -248,6 +254,79 @@ async def test_search_tool_deny_by_default_does_not_load_the_user_when_off( get_user_object.assert_not_awaited() +@pytest.mark.parametrize( + "general_settings", + [{}, {"search_tool_deny_by_default": False}], + ids=["omitted", "false"], +) +@pytest.mark.parametrize( + "user_role", + [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.INTERNAL_USER], + ids=["admin", "internal-user"], +) +@pytest.mark.parametrize( + "key_tools, expected", + [ + pytest.param(None, None, id="unrestricted-key"), + pytest.param(["search-a"], None, id="key-grants-search-a"), + pytest.param(["search-b"], _LEGACY, id="key-excludes-search-a"), + ], +) +@pytest.mark.asyncio +async def test_dashboard_search_tool_grants_without_deny_by_default( + general_settings: Mapping[str, object], + user_role: LitellmUserRoles, + key_tools: list[str] | None, + expected: ProxyErrorTypes | None, +) -> None: + caller: Final = _caller( + virtual_key=False, + key_tools=key_tools, + team_id=UI_TEAM_ID, + user_role=user_role, + ) + load_team, load_team_mock = _missing_dashboard_team_loader() + assert await _denied_by(general_settings, caller, load_team) == expected + load_team_mock.assert_not_awaited() + + +@pytest.mark.parametrize( + "user_role", + [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY], + ids=["admin", "admin-view-only"], +) +@pytest.mark.asyncio +async def test_admin_dashboard_search_tool_grants_with_deny_by_default(user_role: LitellmUserRoles) -> None: + caller: Final = _caller(virtual_key=False, team_id=UI_TEAM_ID, user_role=user_role) + load_team, load_team_mock = _missing_dashboard_team_loader() + assert await _denied_by(_DENY_ON, caller, load_team) is None + load_team_mock.assert_not_awaited() + + +@pytest.mark.parametrize( + "user_id, user_tools, expected", + [ + pytest.param("user-1", ["search-a"], None, id="user-grants-search-a"), + pytest.param("user-1", ["search-b"], _USER, id="user-grants-another-tool"), + pytest.param("user-1", None, _USER, id="user-grant-null"), + pytest.param("user-2", None, _USER, id="no-user-row"), + ], +) +@pytest.mark.asyncio +async def test_internal_dashboard_search_tool_requires_the_user_grant_with_deny_by_default( + cache: UserApiKeyCache, + user_id: str, + user_tools: list[str] | None, + expected: ProxyErrorTypes | None, +) -> None: + if user_id == "user-1": + _cache_user(cache, user_tools) + caller: Final = _caller(virtual_key=False, team_id=UI_TEAM_ID, user_id=user_id) + load_team, load_team_mock = _missing_dashboard_team_loader() + assert await _denied_by(_DENY_ON, caller, load_team) == expected + load_team_mock.assert_not_awaited() + + @pytest.mark.parametrize( "general_settings, caller, expected", [ @@ -257,6 +336,24 @@ async def test_search_tool_deny_by_default_does_not_load_the_user_when_off( pytest.param(_DENY_ON, _caller(virtual_key=False), _USER, id="keyless user"), pytest.param(_DENY_ON, _caller(user_role=LitellmUserRoles.PROXY_ADMIN), _KEY, id="proxy admin key"), pytest.param(_DENY_ON, _caller(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS), None, id="master key"), + pytest.param( + _DENY_ON, + _caller(virtual_key=False, team_id=UI_TEAM_ID, user_role=LitellmUserRoles.INTERNAL_USER), + _USER, + id="internal dashboard user", + ), + pytest.param( + _DENY_ON, + _caller(virtual_key=False, team_id=UI_TEAM_ID, user_role=LitellmUserRoles.PROXY_ADMIN), + None, + id="admin dashboard", + ), + pytest.param( + _DENY_ON, + _caller(virtual_key=False, team_id=UI_TEAM_ID, user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY), + None, + id="admin view-only dashboard", + ), ], ) def test_unregistered_search_fallback_follows_search_tool_deny_by_default( @@ -268,3 +365,12 @@ def test_unregistered_search_fallback_follows_search_tool_deny_by_default( with pytest.raises(ProxyException) as exc_info: check_unregistered_search_fallback(caller, general_settings) assert (exc_info.value.type, exc_info.value.code) == (expected, "403") + + +@pytest.mark.asyncio +async def test_search_preserves_missing_real_team_error_without_deny_by_default() -> None: + missing_team: Final = HTTPException(status_code=404, detail="Team doesn't exist in db. Team=team-1") + load_team: Final = AsyncMock(side_effect=missing_team) + with pytest.raises(HTTPException) as exc_info: + await can_caller_call_search_tool("search-a", _caller(team_id="team-1"), {}, load_team) + assert exc_info.value is missing_team diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 108f7ce2983..2d9c781b0d1 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -7724,7 +7724,8 @@ async def test_user_api_key_auth_does_not_overwrite_end_user_id_set_by_builder() @pytest.mark.asyncio -async def test_user_api_key_auth_authenticates_before_raising_malformed_body_error(): +@pytest.mark.parametrize("body", [b'{}{"model": "gpt-4o"}', b"[]", b"null", b"[1e400]"]) +async def test_user_api_key_auth_authenticates_before_raising_malformed_body_error(body: bytes) -> None: """Regression (LIT-4780): a body that fails to parse must still be authenticated first, so the rejected request's trace carries the caller's key / team / user identity instead of an anonymous root span. The parse error is re-raised @@ -7742,10 +7743,11 @@ async def test_user_api_key_auth_authenticates_before_raising_malformed_body_err "headers": [(b"content-type", b"application/json")], "method": "POST", "path": "/chat/completions", + "query_string": b"probe=1", } ) request._url = URL(url="/chat/completions") - request._body = b'{}{"model": "gpt-4o"}' + request._body = body attrs = _proxy_attrs_for_centralized_checks(user_custom_auth=None) originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py index cd5820fa2fd..354feb0bab1 100644 --- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py @@ -64,6 +64,100 @@ def _starlette_request( return Request(scope, receive) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body", + [b"[]", b'[["model", "test"]]', b"null", b'"s"', b"1", b"true", b"false"], +) +@pytest.mark.parametrize("path", ["/v1/messages", "/vertex_ai/v1/messages", "/lens/datasets"]) +async def test_read_request_body_rejects_non_object_json(body: bytes, path: str) -> None: + request: Final = _starlette_request(body, "application/json", path) + + for _ in range(2): + with pytest.raises(ProxyException) as exc_info: + await read_request_body(request) + assert exc_info.value.code == "400" + assert exc_info.value.type == "invalid_request_error" + assert exc_info.value.param == "request_body" + assert exc_info.value.message == "Invalid JSON payload: expected an object" + assert _safe_get_request_parsed_body(request) is None + assert await request.body() == body + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [b"[1e400]", b"1e400", b"[NaN]"]) +async def test_read_request_body_rejects_non_objects_from_json_fallback(body: bytes) -> None: + with pytest.raises(ProxyException) as exc_info: + await read_request_body(_starlette_request(body, "application/json")) + assert exc_info.value.code == "400" + assert exc_info.value.message == "Invalid JSON payload: expected an object" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body, invalid_unicode", + [ + (b"[]", False), + (b"null", False), + (b'"value"', False), + (b"[1e400]", False), + (b'"\\ud800"', True), + (b'["\\ud800"]', True), + (b'{"value":"\\ud800"}', True), + ], +) +@pytest.mark.parametrize("dispatch", ["custom", "provider", "lens"]) +async def test_pass_through_body_validation_preserves_bytes(dispatch: str, body: bytes, invalid_unicode: bool) -> None: + from fastapi.routing import APIRoute + from litellm.proxy.lens.adapter import lens_request + from litellm.proxy.lens.buffering import LensRoute + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import vertex_proxy_route + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import create_pass_through_route + + path: Final = {"custom": "/v1/messages", "provider": "/vertex_ai/predict", "lens": "/lens/datasets"}[dispatch] + route: Final = { + "custom": APIRoute(path, create_pass_through_route(path, "https://upstream.test"), methods=["POST"]), + "provider": APIRoute("/vertex_ai/{endpoint:path}", vertex_proxy_route, methods=["POST"]), + "lens": LensRoute("/lens/{path:path}", lens_request, methods=["POST"]), + }[dispatch] + incoming: Final = _starlette_request(body, "application/json", path) + _, dispatched_scope = route.matches(incoming.scope) + request: Final = Request({**incoming.scope, **dispatched_scope}, incoming.receive) + + for _ in range(2): + if invalid_unicode: + with pytest.raises(ProxyException) as exc_info: + await read_request_body(request) + assert exc_info.value.code == "400" + else: + assert await read_request_body(request) == {} + assert await request.body() == body + + +@pytest.mark.asyncio +@pytest.mark.parametrize("optional", [False, True]) +@pytest.mark.parametrize("body", [b"null", b" \nnull\t", b"[]", b'"value"']) +async def test_null_body_respects_the_dispatched_body_requirement(optional: bool, body: bytes) -> None: + from fastapi.routing import APIRoute + from litellm.proxy.management_endpoints.key_management_endpoints import generate_key_fn + from litellm.proxy.spend_tracking.spend_management_endpoints import global_spend_end_users + + endpoint: Final = global_spend_end_users if optional else generate_key_fn + route: Final = APIRoute("/audit", endpoint, methods=["POST"]) + incoming: Final = _starlette_request(body, "application/json", "/audit") + _, dispatched_scope = route.matches(incoming.scope) + request: Final = Request({**incoming.scope, **dispatched_scope}, incoming.receive) + + if optional and body.strip() == b"null": + assert await read_request_body(request) == {} + assert await request.json() is None + else: + with pytest.raises(ProxyException) as exc_info: + await read_request_body(request) + assert exc_info.value.code == "400" + assert await request.body() == body + + @pytest.mark.asyncio async def test_read_request_body_marks_body_received_once_with_its_size(monkeypatch: pytest.MonkeyPatch): events: list[tuple[str, dict[str, str | int]]] = [] # mutable-ok: recorder for the injected phase_event double @@ -131,14 +225,20 @@ async def test_read_raw_json_body_is_none_for_form_bodies(): @pytest.mark.asyncio @pytest.mark.parametrize("content_type", ["application/x-protobuf", "application/protobuf; charset=binary"]) -async def test_protobuf_body_is_not_parsed_as_json(content_type): - # OTLP trace exports (POST /v1/traces) are binary protobuf; arbitrary bytes like these - # used to hit the JSON surrogate-repair path and fail auth with a 400. - body = b"\n\xa2\x01\n\x1c\n\x0cservice.name\x12\x0c\n\nswarm\xed\xa0\x80\xff" - request = _starlette_request(body, content_type) +@pytest.mark.parametrize( + "body", + [ + b"\n\xa2\x01\n\x1c\n\x0cservice.name\x12\x0c\n\nswarm\xed\xa0\x80\xff", + pytest.param(b"100000000", id="protobuf-also-json-number"), + ], +) +async def test_protobuf_body_is_not_parsed_as_json(content_type: str, body: bytes) -> None: + request: Final = _starlette_request(body, content_type) assert await read_request_body(request) == {} - assert await request.body() == body # body is still readable by the endpoint + assert await read_request_body(request) == {} + assert await read_raw_json_body(request) == body + assert await request.body() == body @pytest.mark.asyncio diff --git a/tests/unit/proxy/common_utils/test_upsert_budget_membership.py b/tests/unit/proxy/common_utils/test_upsert_budget_membership.py index 1f7154f0a74..4aa1a7f7cb1 100644 --- a/tests/unit/proxy/common_utils/test_upsert_budget_membership.py +++ b/tests/unit/proxy/common_utils/test_upsert_budget_membership.py @@ -267,10 +267,17 @@ async def test_create_from_temp_pair_never_snapshots_team_default(mock_tx, fake_ @pytest.mark.asyncio -async def test_temp_pair_on_shared_default_member_creates_bare_row(mock_tx, fake_user): +async def test_temp_pair_on_shared_default_member_clones_default_with_its_reset_schedule(mock_tx, fake_user): expiry = datetime(2100, 1, 1, tzinfo=timezone.utc) + default_reset_at = datetime(2026, 11, 1, tzinfo=timezone.utc) mock_tx.litellm_budgettable.find_unique = AsyncMock( - return_value=budget_row(budget_id="team-default-budget-1", max_budget=0.4, rpm_limit=10) + return_value=budget_row( + budget_id="team-default-budget-1", + max_budget=0.4, + rpm_limit=10, + budget_duration="30d", + budget_reset_at=default_reset_at, + ) ) await upsert_budget_and_membership( mock_tx, @@ -282,10 +289,14 @@ async def test_temp_pair_on_shared_default_member_creates_bare_row(mock_tx, fake team_default_budget_id="team-default-budget-1", ) - mock_tx.litellm_budgettable.find_unique.assert_not_awaited() mock_tx.litellm_budgettable.update.assert_not_called() + mock_tx.litellm_teammembership.update.assert_not_called() assert stored_budget_row(mock_tx) == { "model_max_budget": "{}", + "max_budget": 0.4, + "rpm_limit": 10, + "budget_duration": "30d", + "budget_reset_at": default_reset_at, "temp_budget_increase": 1.0, "temp_budget_expiry": expiry, } diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 3efbe6475b2..a132ce8eada 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -3,6 +3,7 @@ import asyncio import contextlib import io +import json import logging from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator from types import SimpleNamespace @@ -35,6 +36,7 @@ from litellm.llms.openai.chat.guardrail_translation.handler import ( from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, ) +from litellm.llms.pass_through.guardrail_translation.handler import PassThroughEndpointHandler from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import ( MCPGuardrailTranslationHandler, ) @@ -3053,3 +3055,34 @@ async def test_pre_call_hook_skips_attachment_scan_when_guardrail_has_none(): ) assert len(guardrail.apply_calls) == 1 + + +@pytest.mark.parametrize("hook", ["async_pre_call_hook", "async_moderation_hook"]) +@pytest.mark.asyncio +async def test_pass_through_guardrail_text_is_the_client_body_without_the_injected_key_metadata( + monkeypatch: pytest.MonkeyPatch, hook: str +) -> None: + _patch_translation_mappings(monkeypatch, {CallTypes.pass_through: PassThroughEndpointHandler}) + guardrail: Final = RecordingGuardrail() + body: Final = { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Say hi in one word"}], + "guardrails": ["recording-guardrail"], + } + data: Final = {**body, "guardrail_to_apply": guardrail} + user_api_key_dict: Final = UserAPIKeyAuth( + token="hashed-key", user_email="someone@example.com", team_id="team-1", user_role="proxy_admin" + ) + + if hook == "async_pre_call_hook": + await UnifiedLLMGuardrails().async_pre_call_hook( + user_api_key_dict=user_api_key_dict, cache=DualCache(), data=data, call_type=CallTypes.pass_through.value + ) + else: + await UnifiedLLMGuardrails().async_moderation_hook( + data=data, user_api_key_dict=user_api_key_dict, call_type=CallTypes.pass_through.value + ) + + texts: Final = [call["inputs"]["texts"] for call in guardrail.apply_calls] + assert [[json.loads(text) for text in call_texts] for call_texts in texts] == [[body]], texts + assert data["litellm_metadata"]["user_api_key_user_email"] == "someone@example.com", data["litellm_metadata"] diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 950d1742d49..67b280eb379 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -5437,3 +5437,64 @@ async def test_health_services_endpoint_langfuse_missing_keys_errors(monkeypatch with pytest.raises(ProxyException, match="auth_check failed") as raised: await health_services_endpoint(service="langfuse") assert "connection refused by lf.internal.example" in str(raised.value.message) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("admin", [False, True]) +async def test_unsaved_chatgpt_account_probe_requires_proxy_admin( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, admin: bool +) -> None: + from starlette.requests import Request + from litellm.proxy import proxy_server + + monkeypatch.setenv("CHATGPT_AUTH_FILE", str(tmp_path / "missing-default-auth.json")) + auth_file: Final = tmp_path / "auth.json" + auth_file.write_text( + json.dumps({"access_token": "token-probe", "account_id": "acct-probe", "expires_at": 4102444800}) + ) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + output: Final = { + "id": "msg_probe", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + response: Final = { + "id": "resp_probe", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-6.1-sol", + "output": [output], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + with respx.mock(assert_all_called=False) as upstream: + upstream.post("https://chatgpt.com/backend-api/codex/responses").respond( + 200, + text="data: " + json.dumps({"type": "response.completed", "response": response}) + "\n\n", + headers={"content-type": "text/event-stream"}, + ) + call: Final = health_test_model_connection( + request=Request({"type": "http", "headers": []}), + mode="responses", + litellm_params={"model": "chatgpt/gpt-6.1-sol", "chatgpt_auth_file": str(auth_file)}, + model_info={}, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN if admin else LitellmUserRoles.INTERNAL_USER, token="test-key" + ), + ) + if admin: + result: Final = await call + assert result["status"] == "success" + assert upstream.calls.last.request.headers["Authorization"] == "Bearer token-probe" + assert upstream.calls.last.request.headers["ChatGPT-Account-Id"] == "acct-probe" + assert isinstance(json.loads(upstream.calls.last.request.content)["input"], list) + else: + with pytest.raises(ProxyException) as error: + await call + assert error.value.code == "403" + assert len(upstream.calls) == 0 diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 1ec505362dd..345f37f1db4 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -7,7 +7,7 @@ import contextvars import logging import sys import time -from collections.abc import Iterator, Sequence +from collections.abc import AsyncGenerator, Iterator, Sequence from contextlib import contextmanager from datetime import datetime, timedelta, timezone from typing import Any, Dict, Final, List, Optional @@ -4644,6 +4644,140 @@ async def test_success_event_releases_parallel_slot_v3(monkeypatch): ) +_SSE_KEEPALIVE_CALLBACK_TIME: Final = datetime(2026, 1, 1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("after_ping", [False, True], ids=["fast", "pinged"]) +@pytest.mark.parametrize("fails", [False, True], ids=["success", "failure"]) +@pytest.mark.parametrize("preadmitted", [False, True], ids=["producer-admission", "existing-admission"]) +async def test_sse_keepalive_releases_parallel_capacity( + after_ping: bool, fails: bool, preadmitted: bool +) -> None: + from fastapi.responses import StreamingResponse + + from litellm.proxy.common_request_processing import open_sse_before_first_byte + from litellm.proxy.common_utils.sse_keepalive import SSE_COMMENT_PING_BYTES + + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + api_key: Final = hash_token("sk-keepalive-release") + auth: Final = UserAPIKeyAuth(api_key=api_key, max_parallel_requests=1) + counter: Final = f"{{api_key:{api_key}}}:max_parallel_requests" + + async def request(call_id: str) -> None: + data: Final = {"model": "test-model", "litellm_call_id": call_id} + upstream_answers: Final = asyncio.Event() + if not after_ping: + upstream_answers.set() + + async def admit() -> None: + await handler.async_pre_call_hook( + user_api_key_dict=auth, cache=cache, data=data, call_type="acompletion" + ) + + async def failure(exc: Exception) -> None: + await handler.async_post_call_failure_hook( + request_data=data, original_exception=exc, user_api_key_dict=auth + ) + + async def stream() -> AsyncGenerator[bytes, None]: + yield b"data: [DONE]\n\n" + await handler.async_log_success_event( + kwargs={ + "litellm_call_id": call_id, + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + }, + response_obj=ModelResponse(usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2)), + start_time=_SSE_KEEPALIVE_CALLBACK_TIME, + end_time=_SSE_KEEPALIVE_CALLBACK_TIME, + ) + + async def produce() -> StreamingResponse: + if not preadmitted: + await admit() + await upstream_answers.wait() + if fails: + raise HTTPException(status_code=502, detail="upstream failed") + return StreamingResponse(stream()) + + if preadmitted: + await admit() + response_call: Final = open_sse_before_first_byte( + produce(), 0.001 if after_ping else 60.0, on_late_failure=failure + ) + if fails and not after_ping: + with pytest.raises(HTTPException) as rejected: + await response_call + assert rejected.value.status_code == 502 + await failure(rejected.value) + return + response: Final = await response_call + first: Final = await anext(response.body_iterator) + if after_ping: + upstream_answers.set() + rest: Final = tuple([chunk async for chunk in response.body_iterator]) + assert first == (SSE_COMMENT_PING_BYTES if after_ping else b"data: [DONE]\n\n") + if fails: + error_frame: Final = rest[-2] + assert isinstance(error_frame, bytes) and b"upstream failed" in error_frame + assert rest[-1] == b"data: [DONE]\n\n" + elif after_ping: + assert rest[-1] == b"data: [DONE]\n\n" + else: + assert rest == () + + for index in range(3): + await contextvars.Context().run(asyncio.create_task, request(f"keepalive-{index}")) + assert handler._gauge_in_flight_from_cache_value(await cache.async_get_cache(counter)) == 0 + + +@pytest.mark.asyncio +async def test_sse_keepalive_rejects_overlap_and_reuses_completed_capacity() -> None: + from fastapi.responses import StreamingResponse + + from litellm.proxy.common_request_processing import open_sse_before_first_byte + + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + api_key: Final = hash_token("sk-keepalive-overlap") + auth: Final = UserAPIKeyAuth(api_key=api_key, max_parallel_requests=1) + admitted: Final = asyncio.Event() + finish: Final = asyncio.Event() + + async def request(call_id: str) -> None: + async def stream() -> AsyncGenerator[bytes, None]: + await finish.wait() + yield b"data: [DONE]\n\n" + await handler.async_post_call_success_hook( + data={"litellm_call_id": call_id}, user_api_key_dict=auth, response=ModelResponse() + ) + + async def produce() -> StreamingResponse: + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": "test-model", "litellm_call_id": call_id}, + call_type="acompletion", + ) + admitted.set() + return StreamingResponse(stream()) + + response: Final = await open_sse_before_first_byte(produce(), 60.0) + assert tuple([chunk async for chunk in response.body_iterator]) == (b"data: [DONE]\n\n",) + + first: Final = contextvars.Context().run(asyncio.create_task, request("first")) + await admitted.wait() + try: + with pytest.raises(HTTPException) as rejected: + await contextvars.Context().run(asyncio.create_task, request("overlapping")) + assert rejected.value.status_code == 429 + finally: + finish.set() + await first + await contextvars.Context().run(asyncio.create_task, request("after-completion")) + + @pytest.mark.asyncio async def test_read_only_gauge_check_counts_without_acquiring_v3(): """ diff --git a/tests/unit/proxy/lens/test_buffering.py b/tests/unit/proxy/lens/test_buffering.py index b544b70d36b..2b868e72096 100644 --- a/tests/unit/proxy/lens/test_buffering.py +++ b/tests/unit/proxy/lens/test_buffering.py @@ -249,7 +249,10 @@ async def test_admission_releases_upload_on_disconnect_or_cancellation(cancel: b @pytest.mark.asyncio @pytest.mark.parametrize("reject_auth", [False, True]) -async def test_admitted_bytes_survive_auth_parsing_and_remain_reserved_until_sent(reject_auth: bool) -> None: +@pytest.mark.parametrize("body", [b" {} ", b"[]", b"null", b'"s"']) +async def test_admitted_bytes_survive_auth_parsing_and_remain_reserved_until_sent( + reject_auth: bool, body: bytes +) -> None: app: Final = FastAPI() router: Final = APIRouter(route_class=LensRoute) @@ -264,9 +267,11 @@ async def test_admitted_bytes_survive_auth_parsing_and_remain_reserved_until_sen if reject_auth: raise HTTPException(401, "Invalid key") - async with httpx.AsyncClient( - transport=httpx.MockTransport(lambda _: httpx.Response(200, content=b"result")) - ) as upstream: + def upstream_response(incoming: httpx.Request) -> httpx.Response: + assert incoming.content == body + return httpx.Response(200, content=b"result") + + async with httpx.AsyncClient(transport=httpx.MockTransport(upstream_response)) as upstream: @router.post("/lens/datasets") async def endpoint(request: Request, auth: Annotated[None, Depends(auth_reader)]) -> Response: @@ -284,14 +289,14 @@ async def test_admitted_bytes_survive_auth_parsing_and_remain_reserved_until_sen sending.set() await finish.wait() - incoming: Final = request(b" {} ") + incoming: Final = request(body) task: Final = asyncio.create_task(app(incoming.scope, incoming.receive, send)) try: await asyncio.wait_for(sending.wait(), 2) async with httpx.AsyncClient( transport=httpx.ASGITransport(app=app), base_url="http://gateway.test" ) as client: - blocked: Final = await client.post("/lens/datasets", content=b" {} " * 2) + blocked: Final = await client.post("/lens/datasets", content=b" " * (11 - len(body))) unrelated: Final = await client.post("/ordinary", content=b"ordinary body") assert blocked.status_code == 503 assert unrelated.content == b"ordinary body" diff --git a/tests/unit/proxy/management_endpoints/test_key_budget_change.py b/tests/unit/proxy/management_endpoints/test_key_budget_change.py index baf8d870117..484f8638513 100644 --- a/tests/unit/proxy/management_endpoints/test_key_budget_change.py +++ b/tests/unit/proxy/management_endpoints/test_key_budget_change.py @@ -45,6 +45,7 @@ def _key( max_budget: float | None = None, budget_limits: Sequence[Mapping[str, object]] | None = None, budget_id: str | None = None, + budget_duration: str | None = None, ) -> LiteLLM_VerificationToken: return LiteLLM_VerificationToken( token="hashed-owner-key", @@ -54,6 +55,7 @@ def _key( max_budget=max_budget, budget_limits=None if budget_limits is None else [dict(window) for window in budget_limits], budget_id=budget_id, + budget_duration=budget_duration, metadata={}, ) @@ -127,6 +129,56 @@ def _window(duration: str, cap: float) -> Mapping[str, object]: (UpdateKeyRequest(key="sk-1", spend=0.0), _key(), KeyBudgetAdminOnly()), (UpdateKeyRequest(key="sk-1", soft_budget=1.0), _key(), KeyBudgetAdminOnly()), (UpdateKeyRequest(key="sk-1", soft_budget=None, max_budget=5.0), _key(max_budget=10.0), KeyBudgetAdminOnly()), + ( + UpdateKeyRequest(key="sk-1", budget_duration="1s"), + _key(max_budget=10.0, budget_duration="30d"), + KeyBudgetLoosened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_duration="60d"), + _key(max_budget=10.0, budget_duration="30d"), + KeyBudgetLoosened(), + ), + (UpdateKeyRequest(key="sk-1", budget_duration="1d"), _key(max_budget=10.0), KeyBudgetLoosened()), + ( + UpdateKeyRequest(key="sk-1", budget_duration=None), + _key(max_budget=10.0, budget_duration="30d"), + KeyBudgetTightened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_duration="30d"), + _key(max_budget=10.0, budget_duration="30d"), + KeyBudgetUnchanged(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_duration="30d"), + _key(max_budget=10.0, budget_duration="Monthly"), + KeyBudgetUnchanged(), + ), + (UpdateKeyRequest(key="sk-1", budget_duration="1s"), _key(budget_duration="30d"), KeyBudgetUnchanged()), + (UpdateKeyRequest(key="sk-1", max_budget=5.0, budget_duration="1s"), _key(), KeyBudgetTightened()), + ( + UpdateKeyRequest(key="sk-1", budget_duration="1s"), + _key(budget_id="shared-tier-10", budget_duration="30d"), + KeyBudgetAdminOnly(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_duration=None), + _key(budget_id="shared-tier-10", budget_duration="30d"), + KeyBudgetAdminOnly(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_id="shared-tier-100"), + _key(budget_id="shared-tier-10"), + KeyBudgetAdminOnly(), + ), + (UpdateKeyRequest(key="sk-1", budget_id=None), _key(budget_id="shared-tier-10"), KeyBudgetAdminOnly()), + (UpdateKeyRequest(key="sk-1", budget_id="shared-tier-10"), _key(), KeyBudgetAdminOnly()), + ( + UpdateKeyRequest(key="sk-1", budget_id="shared-tier-10"), + _key(budget_id="shared-tier-10"), + KeyBudgetUnchanged(), + ), ], ) def test_classify_key_budget_change( @@ -465,3 +517,60 @@ async def test_owner_cannot_clear_max_budget_by_default(monkeypatch: pytest.Monk existing=_key(max_budget=10.0), ) assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("policy", "data", "existing"), + [ + (None, UpdateKeyRequest(key="sk-1", budget_duration="1s"), _key(max_budget=10.0, budget_duration="30d")), + ( + "lower_only", + UpdateKeyRequest(key="sk-1", budget_duration="1s"), + _key(max_budget=10.0, budget_duration="30d"), + ), + ("ceiling", UpdateKeyRequest(key="sk-1", budget_id="shared-tier-100"), _key(budget_id="shared-tier-10")), + ("ceiling", UpdateKeyRequest(key="sk-1", budget_duration="1s"), _key(budget_id="shared-tier-10")), + ], +) +async def test_owner_cannot_restart_or_relink_a_key_budget_without_admin( + monkeypatch: pytest.MonkeyPatch, policy: str | None, data: UpdateKeyRequest, existing: LiteLLM_VerificationToken +) -> None: + with pytest.raises(HTTPException) as exc: + await _update(monkeypatch, policy=policy, data=data, existing=existing) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("policy", "data", "existing"), + [ + (None, UpdateKeyRequest(key="sk-1", budget_duration="1d"), _key(budget_duration="30d")), + (None, UpdateKeyRequest(key="sk-1", budget_duration="24h"), _key(max_budget=10.0, budget_duration="daily")), + ( + "lower_only", + UpdateKeyRequest(key="sk-1", budget_duration=None), + _key(max_budget=10.0, budget_duration="30d"), + ), + ("ceiling", UpdateKeyRequest(key="sk-1", budget_duration="1s"), _key(max_budget=10.0, budget_duration="30d")), + ], +) +async def test_owner_changes_budget_duration_when_it_cannot_restart_a_cap_or_policy_allows( + monkeypatch: pytest.MonkeyPatch, policy: str | None, data: UpdateKeyRequest, existing: LiteLLM_VerificationToken +) -> None: + assert await _denial(monkeypatch, policy=policy, data=data, existing=existing) is None + + +@pytest.mark.asyncio +async def test_team_member_cannot_restart_a_team_key_budget_created_by_someone_else( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy=None, + data=UpdateKeyRequest(key="sk-1", budget_duration="1s"), + existing=_key(created_by=_TEAM_ADMIN, team_id=_TEAM, max_budget=10.0, budget_duration="30d"), + team=_team(["/key/update"]), + ) + assert exc.value.status_code == 403 diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 303b8e060ed..9ff2a6de9dd 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -3672,6 +3672,23 @@ class TestUpdateDBModelKeepsLegacyDropParams: assert decrypt_value_helper(value=stored["drop_params"], key="drop_params") == "true" +@pytest.mark.parametrize( + "payload", + [ + {"litellm_params_to_remove": ["model"]}, + {"litellm_params_to_remove": ["temperature"], "litellm_params": {"temperature": 0}}, + {"litellm_params_to_remove": ["temperature"], "litellm_params": {"temperature": None}}, + ], +) +def test_model_param_removal_rejects_required_and_conflicting_fields(payload: Mapping[str, object]) -> None: + from pydantic import ValidationError + + from litellm.types.proxy.management_endpoints.model_management_endpoints import PatchModelRequest + + with pytest.raises(ValidationError): + PatchModelRequest.model_validate(payload) + + def _build_db_model_with_pricing(): """Wildcard deployment with custom pricing in litellm_params; Deployment.__init__ mirrors SPECIAL_MODEL_INFO_PARAMS into model_info, so both blobs hold the rate.""" @@ -7843,6 +7860,55 @@ class TestTeamMemberAutoRouterWrites: "jev_classifier_config" if legacy else "opensource_classifier_config": classifier, } + @pytest.mark.asyncio + @pytest.mark.parametrize( + "stored_params,removed,denied_param", + [ + ({"litellm_credential_name": "stored-credential"}, "litellm_credential_name", "litellm_credential_name"), + ({"aws_session_tags": [{"Key": "scope", "Value": "restricted"}]}, "aws_session_tags", "aws_session_tags"), + ({"api_key": "oidc/file//var/run/token", "api_base": "https://provider.test"}, "api_base", "api_key"), + ({"anthropic_issuer_url": "https://issuer.test"}, "anthropic_issuer_url", "anthropic_issuer_url"), + ], + ) + async def test_team_admin_cannot_remove_admin_owned_model_params( + self, stored_params: Mapping[str, object], removed: str, denied_param: str + ) -> None: + from litellm.types.proxy.management_endpoints.model_management_endpoints import PatchModelRequest + + row: Final = self._row().model_copy( + update={"litellm_params": {"model": "openai/test-model", **stored_params}} + ) + team: Final = self._team().model_copy(update={"members_with_roles": [Member(user_id="owner", role="admin")]}) + database: Final = self._database(team, row) + database.db.litellm_credentialstable.find_unique = AsyncMock(return_value=None) + actor: Final = UserAPIKeyAuth(user_id="owner", user_role=LitellmUserRoles.INTERNAL_USER) + + with self._environment(database, row), pytest.raises(ProxyException) as denied: + await patch_model(row.model_id, PatchModelRequest(litellm_params_to_remove=(removed,)), actor) + + assert denied.value.code == "403" + assert denied.value.param == denied_param + database.db.litellm_proxymodeltable.update.assert_not_awaited() + + @pytest.mark.asyncio + async def test_param_removal_cannot_leave_a_strategy_router_without_required_configuration(self) -> None: + from litellm.types.proxy.management_endpoints.model_management_endpoints import PatchModelRequest + + row: Final = self._row() + database: Final = self._database(self._team(), row) + actor: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + + with self._environment(database, row), pytest.raises(ProxyException) as denied: + await patch_model( + row.model_id, + PatchModelRequest(litellm_params_to_remove=("complexity_router_config", "complexity_router_default_model")), + actor, + ) + + assert denied.value.code == "400" + assert "requires" in denied.value.message + database.db.litellm_proxymodeltable.update.assert_not_awaited() + @pytest.mark.asyncio @pytest.mark.parametrize("team_id", [None, "member-team"]) @pytest.mark.parametrize( @@ -8933,7 +8999,14 @@ class TestNonAdminCannotPersistWifFieldsOnModel: mock_prisma.db.litellm_proxymodeltable.update.assert_not_called() @pytest.mark.asyncio - async def test_patch_model_non_admin_cannot_set_wif_field(self): + @pytest.mark.parametrize( + "field,value", + [ + ("anthropic_keycloak_token_url", "https://identity.example.com/token"), + ("chatgpt_auth_file", "/tokens/account-b/auth.json"), + ], + ) + async def test_patch_model_non_admin_cannot_set_wif_field(self, field: str, value: str) -> None: from litellm.proxy.management_endpoints.model_management_endpoints import ( patch_model, ) @@ -8979,13 +9052,13 @@ class TestNonAdminCannotPersistWifFieldsOnModel: model_id="m1", patch_data=updateDeployment( litellm_params=updateLiteLLMParams( - anthropic_keycloak_token_url="https://attacker.example/token", + **{field: value}, ) ), user_api_key_dict=non_admin, ) err = exc_info.value - assert getattr(err, "param", "") == "anthropic_keycloak_token_url" + assert getattr(err, "param", "") == field mock_prisma.db.litellm_proxymodeltable.update.assert_not_called() @pytest.mark.asyncio @@ -9044,7 +9117,14 @@ class TestNonAdminCannotPersistWifFieldsOnModel: mock_prisma.db.litellm_proxymodeltable.update.assert_not_called() @pytest.mark.asyncio - async def test_patch_model_admin_can_set_wif_field(self): + @pytest.mark.parametrize( + "field,value", + [ + ("anthropic_keycloak_token_url", "https://identity.example.com/token"), + ("chatgpt_auth_file", "/tokens/account-b/auth.json"), + ], + ) + async def test_patch_model_admin_can_set_wif_field(self, field: str, value: str) -> None: from litellm.proxy.management_endpoints.model_management_endpoints import ( patch_model, ) @@ -9095,7 +9175,7 @@ class TestNonAdminCannotPersistWifFieldsOnModel: model_id="m1", patch_data=updateDeployment( litellm_params=updateLiteLLMParams( - anthropic_keycloak_token_url="https://keycloak.internal/token", + **{field: value}, ) ), user_api_key_dict=admin, diff --git a/tests/unit/proxy/management_endpoints/test_organization_endpoints.py b/tests/unit/proxy/management_endpoints/test_organization_endpoints.py index a5cae02ea10..16c59aa20b9 100644 --- a/tests/unit/proxy/management_endpoints/test_organization_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_organization_endpoints.py @@ -1615,3 +1615,68 @@ async def test_delete_organization_is_refused_while_one_of_its_teams_holds_a_ptu prisma_client.db.litellm_teamtable.delete_many.assert_not_called() prisma_client.db.litellm_verificationtoken.delete_many.assert_not_called() prisma_client.db.litellm_organizationtable.delete.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("budget_duration", ["30d", None]) +async def test_new_organization_schedules_budget_reset_from_duration(monkeypatch, budget_duration): + """/organization/new with an inline budget_duration must write budget_reset_at on the auto-created + budget row, the way team create and /budget/new do, or the org budget never resets and the UI + cannot show the current budget period.""" + from datetime import datetime + + from litellm.proxy._types import LitellmUserRoles, NewOrganizationRequest, UserAPIKeyAuth + from litellm.proxy.management_endpoints.organization_endpoints import new_organization + from litellm.proxy.utils import PrismaClient + + prisma_client = MagicMock() + prisma_client.jsonify_object = MagicMock(side_effect=lambda data: PrismaClient.jsonify_object(prisma_client, data)) + prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_budgettable.create = AsyncMock(return_value=MagicMock(budget_id="budget-1")) + prisma_client.db.litellm_organizationtable.create = AsyncMock(return_value={"organization_id": "org-1"}) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", MagicMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True, raising=False) + + await new_organization( + data=NewOrganizationRequest(organization_alias="org", max_budget=100, budget_duration=budget_duration), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + budget_write = prisma_client.db.litellm_budgettable.create.await_args.kwargs["data"] + if budget_duration is None: + assert "budget_reset_at" not in budget_write, budget_write + return + reset_at = budget_write["budget_reset_at"] + assert isinstance(reset_at, datetime) and reset_at.tzinfo is not None, budget_write + assert budget_write["budget_duration"] == budget_duration + + +def test_organization_info_response_keeps_budget_reset_at(): + """/organization/info must return the budget row's budget_reset_at so the dashboard can show + which budget period the org's spend covers.""" + from datetime import datetime, timezone + + from litellm.proxy._types import LiteLLM_OrganizationTableWithMembers + + reset_at = datetime(2026, 11, 1, tzinfo=timezone.utc) + created_at = datetime(2026, 10, 1, tzinfo=timezone.utc) + response = LiteLLM_OrganizationTableWithMembers( + organization_id="org-1", + organization_alias="org", + budget_id="budget-1", + models=[], + created_by="admin", + updated_by="admin", + created_at=created_at, + updated_at=created_at, + litellm_budget_table={ + "budget_id": "budget-1", + "max_budget": 100.0, + "budget_duration": "30d", + "budget_reset_at": reset_at, + "created_at": created_at, + }, + ) + + assert response.model_dump()["litellm_budget_table"]["budget_reset_at"] == reset_at diff --git a/tests/unit/proxy/management_endpoints/test_ptu_consumption.py b/tests/unit/proxy/management_endpoints/test_ptu_consumption.py index a446afbd809..065ec93dcd0 100644 --- a/tests/unit/proxy/management_endpoints/test_ptu_consumption.py +++ b/tests/unit/proxy/management_endpoints/test_ptu_consumption.py @@ -232,14 +232,10 @@ def test_rows_keyed_by_a_name_a_wildcard_route_serves_are_sized_by_its_shared_de assert attached.results[0].breakdown.model_groups["ptu-chat"].metrics.ptu_hours == pytest.approx(1.0) -@pytest.mark.parametrize( - ("catch_all_routes", "expected_ptu_hours"), [(False, 1.0), (True, 0.0)], ids=["team-wildcard", "behind-catch-all"] -) -def test_a_teams_wildcard_reservation_reads_ptu_hours_only_while_the_router_serves_from_it( - monkeypatch, catch_all_routes, expected_ptu_hours -): - """The router tries the proxy-wide wildcards before a team's own, so a team's `ptu-*` - reservation behind an open `*` route never serves `ptu-chat`, and its row reads no PTU-hours.""" +@pytest.mark.parametrize("catch_all_routes", [False, True], ids=["team-wildcard", "beside-catch-all"]) +def test_a_teams_wildcard_reservation_reads_ptu_hours_even_beside_a_proxy_wide_catch_all(monkeypatch, catch_all_routes): + """The router tries a team's own wildcard before the proxy-wide ones, so a team's `ptu-*` + reservation serves `ptu-chat` whether or not an open `*` route exists, and its row reads its PTU-hours.""" monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True") reservation: Final = { "model_name": "ptu-team-r", @@ -264,9 +260,7 @@ def test_a_teams_wildcard_reservation_reads_ptu_hours_only_while_the_router_serv attached: Final = with_ptu_consumption(_response(_day("2026-09-23", {"ptu-chat": one_hour})), router, "team-r") - assert attached.results[0].breakdown.model_groups["ptu-chat"].metrics.ptu_hours == pytest.approx( - expected_ptu_hours - ) + assert attached.results[0].breakdown.model_groups["ptu-chat"].metrics.ptu_hours == pytest.approx(1.0) def _mixed_ptu_router() -> Router: diff --git a/tests/unit/proxy/management_endpoints/test_ptu_model_settings.py b/tests/unit/proxy/management_endpoints/test_ptu_model_settings.py index a319797a618..efb84ee8f3a 100644 --- a/tests/unit/proxy/management_endpoints/test_ptu_model_settings.py +++ b/tests/unit/proxy/management_endpoints/test_ptu_model_settings.py @@ -40,6 +40,7 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR from litellm.repositories.chunked_in import IN_LIST_CHUNK_SIZE from litellm.router import Router +from litellm.types.proxy.management_endpoints.model_management_endpoints import PatchModelRequest from litellm.types.router import ( SPECIAL_MODEL_INFO_PARAMS, Deployment, @@ -1205,6 +1206,80 @@ class TestPtuDeploymentsAreNotBilledPerToken: for blob in ("model_info", "litellm_params"): assert json.loads(written[blob])["input_cost_per_token"] == 0, blob + @pytest.mark.parametrize("field,activating", [ + ("input_cost_per_token", False), ("cache_read_input_token_cost", False), + ("input_cost_per_second", False), ("tiered_pricing", False), ("search_context_cost_per_query", False), + ("input_cost_per_token", True), ("cache_read_input_token_cost", True), + ("tiered_pricing", True), ("search_context_cost_per_query", True), + ]) + def test_removing_ptu_managed_pricing_is_rejected(self, field: str, activating: bool) -> None: + original: Final = _deployment_without_ptu() if activating else _deployment_with_stored_ptu() + configured: Final = original.model_copy(update={ + "litellm_params": original.litellm_params.model_copy(update={"input_cost_per_second": 0.0}), + }) + with pytest.raises(HTTPException) as exc: + update_db_model( + db_model=configured if activating else _ptu_priced_deployment(configured), + updated_patch=PatchModelRequest( + litellm_params_to_remove=(field,), + model_info=_deployment_with_stored_ptu().model_info if activating else None, + ), + ) + assert exc.value.status_code == 400 + assert field in exc.value.detail + assert "ptu_count and cost_per_ptu_per_hour" in exc.value.detail + + @pytest.mark.parametrize("mode,field", [ + ("extra", "temperature"), ("disabled", "input_cost_per_token"), + ("disabled", "input_cost_per_second"), ("cleared", "input_cost_per_token"), + ("cleared", "input_cost_per_second"), ("activating", "input_cost_per_second"), + ("ordinary", "input_cost_per_second"), + ("ordinary", "input_cost_per_token_above_123k_tokens"), + ("ordinary", "regional_endpoint_uplift_multiplier"), + ]) + def test_allowed_ptu_removals_stay_absent_after_another_save( + self, monkeypatch: pytest.MonkeyPatch, mode: str, field: str + ) -> None: + original: Final = ( + _deployment_without_ptu() if mode in ("activating", "ordinary") + else _ptu_priced_deployment(_deployment_with_stored_ptu()) + ) + rate: Final = 0.5 if mode in ("activating", "ordinary") else 0.0 + price_field: Final = field if mode == "ordinary" else "input_cost_per_second" + configured: Final = original.model_copy(update={ + "litellm_params": original.litellm_params.model_copy(update={ + "temperature": 0.4, price_field: rate, + }), + "model_info": original.model_info.model_copy(update={price_field: rate}), + }) + if mode == "disabled": + monkeypatch.delenv(PTU_COST_ATTRIBUTION_ENV_VAR) + incoming_info: Final = ( + ModelInfo(ptu_count=None, cost_per_ptu_per_hour=None) if mode == "cleared" + else _deployment_with_stored_ptu().model_info if mode == "activating" else ModelInfo() + ) + patch: Final = PatchModelRequest( + litellm_params_to_remove=(field,), + model_info=incoming_info.model_copy(update={price_field: rate}), + ) + written: Final = update_db_model(db_model=configured, updated_patch=patch) + assert json.loads(written["model_info"]).get("ptu_count") == ( + None if mode in ("cleared", "ordinary") else 15 + ) + reloaded: Final = Deployment.model_validate({ + "model_name": written["model_name"], "litellm_params": json.loads(written["litellm_params"]), + "model_info": json.loads(written["model_info"]), + }) + saved_again: Final = update_db_model( + db_model=reloaded, updated_patch=updateDeployment( + model_name="renamed", model_info=ModelInfo.model_validate({price_field: rate}), + ) + ) + retried: Final = update_db_model(db_model=reloaded, updated_patch=patch) + for result in (written, saved_again, retried): + assert field not in json.loads(result["litellm_params"]) + assert field not in json.loads(result["model_info"]) + def test_a_patch_pricing_a_ptu_deployment_is_refused(self): with pytest.raises(HTTPException) as exc: update_db_model( diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py new file mode 100644 index 00000000000..354c65776dc --- /dev/null +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -0,0 +1,185 @@ +import asyncio +import json +from collections.abc import Iterator +from datetime import datetime +from typing import Final + +import httpx +import pytest +import respx +from fastapi.testclient import TestClient + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy import proxy_server +from tests._master_key import MASTER_KEY + +_BASE: Final = "https://chatgpt.com/backend-api/codex" +_TOKEN: Final = "test-subscription-token" +_HEADERS: Final = {"authorization": f"Bearer {_TOKEN}", "x-litellm-api-key": MASTER_KEY} + + +@pytest.fixture +def client(monkeypatch: pytest.MonkeyPatch, httpx_transport: None) -> Iterator[TestClient]: + monkeypatch.setattr(proxy_server, "master_key", MASTER_KEY) + yield TestClient(proxy_server.app) + + +def test_native_catalog_preserves_account_and_filters_gateway_credentials( + client: TestClient, respx_mock: respx.MockRouter +) -> None: + catalog: Final = {"models": [{"slug": "subscription-model", "supported_reasoning_levels": ["high"]}]} + upstream: Final = respx_mock.get(f"{_BASE}/models", params={"client_version": "1.0.0"}).respond(json=catalog) + response: Final = client.get( + "/backend-api/codex/models?client_version=1.0.0", + headers={ + **_HEADERS, + "chatgpt-account-id": "account-a", + "cookie": "session=test-cookie", + "x-pass-authorization": "Bearer test-injected-token", + "x-pass-x-litellm-api-key": "test-injected-key", + }, + ) + assert response.status_code == 200 + assert response.json() == catalog + assert upstream.calls.last.request.headers["authorization"] == f"Bearer {_TOKEN}" + assert upstream.calls.last.request.headers["chatgpt-account-id"] == "account-a" + assert "x-litellm-api-key" not in upstream.calls.last.request.headers + assert "cookie" not in upstream.calls.last.request.headers + + +@pytest.mark.parametrize("account", ["account-a", "account-b"]) +def test_native_responses_preserves_body_and_stream( + client: TestClient, respx_mock: respx.MockRouter, account: str +) -> None: + body: Final = { + "model": "subscription-model", + "input": [{"role": "user", "content": "Hello"}], + "reasoning": {"effort": "high"}, + "store": False, + "stream": True, + } + stream: Final = ( + b'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"Hello"}\n\n' + ) + upstream: Final = respx_mock.post(f"{_BASE}/responses").respond( + content=stream, headers={"content-type": "text/event-stream"} + ) + response: Final = client.post( + "/backend-api/codex/responses", + json=body, + headers={**_HEADERS, "authorization": f"Bearer test-{account}", "chatgpt-account-id": account}, + ) + assert response.status_code == 200 + assert response.content == stream + assert response.headers["content-type"].startswith("text/event-stream") + assert json.loads(upstream.calls.last.request.content) == body + assert upstream.calls.last.request.headers["authorization"] == f"Bearer test-{account}" + assert upstream.calls.last.request.headers["chatgpt-account-id"] == account + + +@pytest.mark.parametrize("gateway_key", [None, "sk-invalid-test-key"]) +def test_gateway_auth_rejects_before_upstream( + client: TestClient, respx_mock: respx.MockRouter, gateway_key: str | None +) -> None: + headers: Final = { + "authorization": f"Bearer {_TOKEN}", + **({"x-litellm-api-key": gateway_key} if gateway_key else {}), + } + response: Final = client.get("/backend-api/codex/models", headers=headers) + assert response.status_code in (400, 401, 403) + assert not respx_mock.calls + + +@pytest.mark.parametrize("authorization", ["", "Basic test", "Bearer ", f"Bearer {MASTER_KEY}"]) +def test_subscription_credential_required(client: TestClient, respx_mock: respx.MockRouter, authorization: str) -> None: + response: Final = client.get("/backend-api/codex/models", headers={**_HEADERS, "authorization": authorization}) + assert response.status_code == 401 + assert not respx_mock.calls + + +@pytest.mark.parametrize("status", [401, 403, 429]) +def test_provider_errors_do_not_fall_back(client: TestClient, respx_mock: respx.MockRouter, status: int) -> None: + upstream: Final = respx_mock.post(f"{_BASE}/responses").respond( + status, json={"error": {"message": "subscription unavailable"}}, headers={"retry-after": "60"} + ) + response: Final = client.post( + "/backend-api/codex/responses", headers=_HEADERS, json={"model": "subscription-model", "stream": True} + ) + assert response.status_code == status + assert upstream.call_count == 1 + + +@pytest.mark.parametrize("method,path", [("POST", "models"), ("GET", "responses"), ("GET", "deviceauth")]) +def test_only_native_inference_routes_are_forwarded( + client: TestClient, respx_mock: respx.MockRouter, method: str, path: str +) -> None: + response: Final = client.request(method, f"/backend-api/codex/{path}", headers=_HEADERS) + assert response.status_code in (405, 422) + assert not respx_mock.calls + + +class UsageRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.events: Final[asyncio.Queue[dict[str, object]]] = asyncio.Queue() + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + await self.events.put(kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_usage_logged_without_credentials( + monkeypatch: pytest.MonkeyPatch, httpx_transport: None, respx_mock: respx.MockRouter, stream: bool +) -> None: + monkeypatch.setattr(proxy_server, "master_key", MASTER_KEY) + recorder: Final = UsageRecorder() + monkeypatch.setattr(litellm, "_async_success_callback", [recorder]) + litellm.register_model( + { + "chatgpt/subscription-model": { + "litellm_provider": "chatgpt", + "mode": "responses", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + } + ) + result: Final = { + "id": "resp_test", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "subscription-model", + "output": [], + "usage": {"input_tokens": 12, "output_tokens": 3, "total_tokens": 15}, + } + event: Final = {"type": "response.completed", "response": result} + response_body: Final = ( + httpx.Response( + 200, + content=f"event: response.completed\ndata: {json.dumps(event)}\n\n", + headers={"content-type": "text/event-stream"}, + ) + if stream + else httpx.Response(200, json=result) + ) + respx_mock.post(f"{_BASE}/responses").mock(return_value=response_body) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=proxy_server.app), base_url="http://test") as caller: + response: Final = await caller.post( + "/backend-api/codex/responses", + json={"model": "subscription-model", "stream": stream, "input": "Hello"}, + headers={**_HEADERS, "x-pass-authorization": "Bearer test-hidden-token", "cookie": "test-hidden-cookie"}, + ) + assert response.status_code == 200 + logged: Final = await asyncio.wait_for(recorder.events.get(), timeout=5) + payload: Final = logged["standard_logging_object"] + assert isinstance(payload, dict) + assert payload["prompt_tokens"] == 12 + assert payload["completion_tokens"] == 3 + serialized: Final = json.dumps(logged, default=str) + for secret in (_TOKEN, MASTER_KEY, "test-hidden-token", "test-hidden-cookie"): + assert secret not in serialized diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 0186a33dbc4..60043daa402 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -4357,6 +4357,7 @@ class TestVertexRawPredictStreamingClassification: "type": "http", "method": "POST", "path": f"/vertex_ai/{endpoint}", + "path_params": {"endpoint": endpoint}, "headers": [ (b"content-type", b"application/json"), (b"x-litellm-api-key", b"test-key"), diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 2c0e0944edb..9993361d930 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -11,6 +11,7 @@ from __future__ import annotations import asyncio import copy +import json from collections.abc import Callable from contextlib import AbstractContextManager from typing import Final @@ -24,6 +25,7 @@ import litellm from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy import proxy_server +from litellm.proxy._types import LiteLLM_ProxyModelTable from litellm.utils import _invalidate_model_cost_lowercase_map from .conftest import normalize # type: ignore[import-not-found] @@ -206,6 +208,80 @@ def test_v2_model_info_invalid_page_returns_422(client, auth_as, empty_router): assert "detail" in response.json() +@pytest.mark.parametrize("encoded", [False, True]) +def test_deleted_model_params_stay_absent_after_reload_and_another_save( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + mock_prisma: MagicMock, + encoded: bool, +) -> None: + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.proxy.management_endpoints.model_management_endpoints import PatchModelRequest + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment, updateLiteLLMParams + + configured: Final = { + "model": "openai/test-model", "api_key": "synthetic-runtime-secret", + "use_litellm_proxy": True, "use_in_pass_through": True, "use_chat_completions_api": False, + "litellm_image_enabled": True, "timeout": 30, "input_cost_per_token": 0.003, + "team_id": "param-only-team", "temperature": 0, "metadata": {}, + } + original: Final = Deployment( + model_name="editor-model", litellm_params=LiteLLM_Params.model_validate(configured), + model_info=ModelInfo(id="editor-model", team_id="owner-team", access_groups=["retained-group"]), + ) + removals: Final = ("litellm_image_enabled", "use_litellm_proxy", "input_cost_per_token", "team_id") + deleted: Final = update_db_model( + db_model=original, + updated_patch=PatchModelRequest( + litellm_params=updateLiteLLMParams(max_retries=0), litellm_params_to_remove=removals, + model_info=ModelInfo(id="editor-model", input_cost_per_token=0.003), + ), + ) + saved_params: Final = json.loads(deleted["litellm_params"]) + saved_info: Final = json.loads(deleted["model_info"]) + assert saved_params == {**{key: value for key, value in configured.items() if key not in removals}, "max_retries": 0} + assert "input_cost_per_token" not in saved_info + assert saved_info["team_id"] == "owner-team" + assert saved_info["access_groups"] == ["retained-group"] + reloaded: Final = Deployment.model_validate( + {"model_name": original.model_name, "litellm_params": saved_params, "model_info": saved_info} + ) + router: Final = litellm.Router(model_list=[{ + "model_name": "editor-model", "litellm_params": saved_params, + "model_info": {"id": "editor-model", "db_model": True}, + }]) + stored_params: Final = {**saved_params, "api_key": "synthetic-stored-secret"} + mock_prisma.db.litellm_proxymodeltable.find_unique.return_value = LiteLLM_ProxyModelTable( + model_id="editor-model", model_name="editor-model", + litellm_params=json.dumps(stored_params) if encoded else stored_params, + model_info={"id": "editor-model"}, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "prisma_client", mock_prisma) + + with auth_as(): + normal: Final = client.get("/v2/model/info", params={"modelId": "editor-model"}) + editor: Final = client.get("/v2/model/info", params={"modelId": "editor-model", "use_stored_params": "true"}) + + assert normal.status_code == editor.status_code == 200, editor.text + normal_params: Final = normal.json()["data"][0]["litellm_params"] + assert "use_litellm_proxy" in normal_params + assert editor.json()["data"][0]["litellm_params"] == { + key: value for key, value in normal_params.items() if key in saved_params + } + assert editor.json()["data"][0]["litellm_params"]["use_chat_completions_api"] is False + assert "synthetic-runtime-secret" not in editor.text + assert "synthetic-stored-secret" not in editor.text + assert "use_litellm_proxy" in router.model_list[0]["litellm_params"] + + saved_again: Final = update_db_model( + db_model=reloaded, updated_patch=updateDeployment(litellm_params=updateLiteLLMParams(rpm=10, timeout=None)) + ) + persisted: Final = json.loads(saved_again["litellm_params"]) + assert persisted == {**saved_params, "rpm": 10} + + def test_v2_model_info_in_openapi_schema(): """``GET /v2/model/info`` is published in the proxy OpenAPI/Swagger spec.""" from litellm.proxy.proxy_server import get_openapi_schema diff --git a/tests/unit/proxy/response_polling/test_response_polling_handler.py b/tests/unit/proxy/response_polling/test_response_polling_handler.py index 979ab499f3b..63e38ce533a 100644 --- a/tests/unit/proxy/response_polling/test_response_polling_handler.py +++ b/tests/unit/proxy/response_polling/test_response_polling_handler.py @@ -1011,10 +1011,9 @@ class TestPollingConditionChecks: # Create mock router mock_router = Mock() - mock_router.model_name_to_deployment_indices = {"gpt-5": [0]} - mock_router.model_list = [ - {"model_name": "gpt-5", "litellm_params": {"model": "openai/gpt-5"}} - ] + mock_router.deployments_named.side_effect = { + "gpt-5": ({"model_name": "gpt-5", "litellm_params": {"model": "openai/gpt-5"}},) + }.get result = should_use_polling_for_request( background_mode=True, @@ -1033,13 +1032,14 @@ class TestPollingConditionChecks: ) mock_router = Mock() - mock_router.model_name_to_deployment_indices = {"claude-3": [0]} - mock_router.model_list = [ - { - "model_name": "claude-3", - "litellm_params": {"model": "anthropic/claude-3-sonnet"}, - } - ] + mock_router.deployments_named.side_effect = { + "claude-3": ( + { + "model_name": "claude-3", + "litellm_params": {"model": "anthropic/claude-3-sonnet"}, + }, + ) + }.get result = should_use_polling_for_request( background_mode=True, @@ -1183,13 +1183,14 @@ class TestPollingConditionChecks: ) mock_router = Mock() - mock_router.model_name_to_deployment_indices = {"deep-research": [0]} - mock_router.model_list = [ - { - "model_name": "deep-research", - "litellm_params": {"model": "openai/o4-mini-deep-research"}, - } - ] + mock_router.deployments_named.side_effect = { + "deep-research": ( + { + "model_name": "deep-research", + "litellm_params": {"model": "openai/o4-mini-deep-research"}, + }, + ) + }.get # Model alias "deep-research" is in native_background_mode result = should_use_polling_for_request( diff --git a/tests/unit/proxy/test__types.py b/tests/unit/proxy/test__types.py index 68bfe4b34e5..76ca49362b4 100644 --- a/tests/unit/proxy/test__types.py +++ b/tests/unit/proxy/test__types.py @@ -2,23 +2,35 @@ import json from typing import Final import pytest -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from litellm.proxy._types import ( ROLES_WITHIN_ORG, + BudgetNewRequest, ChangePasswordRequest, GenerateKeyRequest, + GenerateKeyResponse, KeyRequest, LiteLLM_AuditLogs, + LiteLLM_BudgetTable, + LiteLLM_EndUserTable, LiteLLM_TeamMembership, + LiteLLM_TeamTable, + LiteLLM_UserTable, LitellmUserRoles, + NewCustomerRequest, NewMCPServerRequest, + NewTeamRequest, NewUserRequest, + NewUserResponse, OrganizationMemberUpdateRequest, + RegenerateKeyRequest, ResetSpendRequest, UpdateKeyRequest, UpdateMCPServerRequest, + UpdateTeamRequest, UpdateUserRequest, + UpdateUserRequestNoUserIDorEmail, UserAPIKeyAuth, ) @@ -213,6 +225,75 @@ def test_a_max_budget_that_was_actually_sent_is_kept(sent): assert GenerateKeyRequest(max_budget=sent).max_budget == sent +RATE_LIMIT_REQUESTS: Final = ( + (GenerateKeyRequest, {}), + (RegenerateKeyRequest, {}), + (UpdateKeyRequest, {"key": "sk-test"}), + (NewUserRequest, {}), + (UpdateUserRequestNoUserIDorEmail, {}), + (UpdateUserRequest, {"user_id": "user-test"}), + (NewTeamRequest, {}), + (UpdateTeamRequest, {"team_id": "team-test"}), + (BudgetNewRequest, {}), + (NewCustomerRequest, {"user_id": "customer-test"}), +) + + +@pytest.mark.parametrize(("request_model", "identifiers"), RATE_LIMIT_REQUESTS) +@pytest.mark.parametrize("field", ("rpm_limit", "tpm_limit")) +@pytest.mark.parametrize("value", (-1, -100)) +def test_management_requests_reject_negative_rate_limits( + request_model: type[BaseModel], identifiers: dict[str, str], field: str, value: int +) -> None: + with pytest.raises(ValidationError) as error: + request_model.model_validate({**identifiers, field: value}) + assert error.value.errors()[0]["loc"] == (field,) + + +@pytest.mark.parametrize(("request_model", "identifiers"), RATE_LIMIT_REQUESTS) +@pytest.mark.parametrize("value", (None, 0, 100)) +def test_management_requests_preserve_explicit_rate_limits( + request_model: type[BaseModel], identifiers: dict[str, str], value: int | None +) -> None: + limits: Final = {"rpm_limit": value, "tpm_limit": value} + request: Final = request_model.model_validate({**identifiers, **limits}) + assert request.model_dump(include={"rpm_limit", "tpm_limit"}, exclude_unset=True) == limits + + +@pytest.mark.parametrize(("request_model", "identifiers"), RATE_LIMIT_REQUESTS) +def test_management_requests_leave_omitted_rate_limits_unset( + request_model: type[BaseModel], identifiers: dict[str, str] +) -> None: + request: Final = request_model.model_validate({**identifiers}) + assert request.model_dump(include={"rpm_limit", "tpm_limit"}) == {"rpm_limit": None, "tpm_limit": None} + assert request.model_dump(include={"rpm_limit", "tpm_limit"}, exclude_unset=True) == {} + + +@pytest.mark.parametrize(("response_model", "identifiers"), ( + (GenerateKeyResponse, {"key": "sk-test"}), + (NewUserResponse, {"key": "sk-test"}), + (LiteLLM_TeamTable, {"team_id": "team-test"}), + (LiteLLM_UserTable, {"user_id": "user-test"}), + (LiteLLM_BudgetTable, {"budget_id": "budget-test"}), +)) +def test_existing_negative_rate_limits_remain_readable( + response_model: type[BaseModel], identifiers: dict[str, str] +) -> None: + limits: Final = {"rpm_limit": -1, "tpm_limit": -100} + response: Final = response_model.model_validate({**identifiers, **limits}) + assert response.model_dump(include={"rpm_limit", "tpm_limit"}) == limits + + +def test_existing_customer_negative_rate_limits_remain_readable() -> None: + customer: Final = LiteLLM_EndUserTable.model_validate({ + "user_id": "customer-test", "blocked": False, + "litellm_budget_table": {"rpm_limit": -1, "tpm_limit": -100}, + }) + assert customer.litellm_budget_table is not None + assert customer.litellm_budget_table.rpm_limit == -1 + assert customer.litellm_budget_table.tpm_limit == -100 + + USER_IDENTIFIER_CASES = ( ({"user_id": "user-1"}, True), ({"user_email": "user@example.com"}, True), diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 64ff8392c02..e03fd9850c1 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -1,4 +1,5 @@ import asyncio +import base64 import copy import json import os @@ -926,6 +927,222 @@ async def test_add_litellm_data_to_request_body_snapshot_excludes_proxy_server_r assert snapshot_body["messages"] == [{"role": "user", "content": "hello"}] +FORWARD_PROVIDER_AUTH_HEADERS: Final = { + "forward_llm_provider_auth_headers": True, + "forward_client_headers_to_llm_api": True, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "sent,request_headers,general_settings,expected", + [ + ({"headers": {"Authorization": "Bearer sk-caller"}}, {}, {}, ("headers",)), + ({"headers": {"X-Trace-Id": "trace-1"}}, {}, {}, ("headers",)), + ({"headers": {}}, {}, {}, ()), + ({"extra_headers": {"API-Key": "caller-azure-key"}}, {}, {}, ("extra_headers",)), + ({"extra_headers": {"OpenAI-Project": "proj_caller"}}, {}, {}, ("extra_headers",)), + ( + {"provider_specific_header": {"custom_llm_provider": "openai", "extra_headers": {"x-api-key": "caller"}}}, + {}, + {}, + ("provider_specific_header",), + ), + ( + { + "provider_specific_header": [ + {"custom_llm_provider": "anthropic", "extra_headers": {"anthropic-beta": "tools"}}, + {"custom_llm_provider": "gemini", "extra_headers": {"X-Goog-Api-Key": "caller"}}, + ] + }, + {}, + {}, + ("provider_specific_header",), + ), + ( + { + "provider_specific_header": { + "custom_llm_provider": "anthropic", + "extra_headers": {"x-caller-trace": "x"}, + } + }, + {}, + {}, + ("provider_specific_header",), + ), + ( + { + "provider_specific_header": { + "custom_llm_provider": "anthropic", + "extra_headers": {"anthropic-version": "2023-06-01", "anthropic-beta": "tools-2024-04-04"}, + } + }, + {}, + {}, + (), + ), + ({"headers": {"Anthropic-Version": "2023-06-01"}}, {}, {}, ()), + ({"organization": "org-caller"}, {}, {}, ("organization",)), + ({"headers": "authorization", "provider_specific_header": "x-api-key"}, {}, {}, ()), + ( + {"api_key": "sk-caller", "headers": {"Ocp-Apim-Subscription-Key": "caller"}}, + {}, + {}, + ("api_key", "headers"), + ), + ({}, {"anthropic-beta": "tools-2024-04-04"}, {}, ()), + ({}, {"anthropic-version": "2023-06-01", "anthropic-beta": "tools-2024-04-04"}, {}, ()), + ({}, {"openai-organization": "org-caller"}, {"forward_openai_org_id": True}, ("organization",)), + ( + {}, + {"x-litellm-api-key": "sk-virtual", "authorization": "Bearer sk-ant-oat01-caller"}, + {}, + ("provider_specific_header",), + ), + ( + {}, + {"authorization": "Bearer sk-virtual", "x-goog-api-key": "AIza-caller"}, + FORWARD_PROVIDER_AUTH_HEADERS, + ("headers",), + ), + ( + {}, + {"authorization": "Bearer sk-virtual", "x-portkey-api-key": "caller-gateway-key"}, + FORWARD_PROVIDER_AUTH_HEADERS, + ("headers",), + ), + ({}, {"authorization": "Bearer sk-virtual"}, FORWARD_PROVIDER_AUTH_HEADERS, ()), + ], + ids=[ + "authorization-in-headers", + "plain-headers", + "empty-headers", + "api-key-in-extra-headers", + "openai-project-in-extra-headers", + "credential-in-provider-specific-header", + "credential-in-one-provider-specific-header-entry", + "plain-provider-specific-header", + "anthropic-version-and-beta-in-provider-specific-header", + "anthropic-version-in-headers", + "organization", + "malformed-header-fields", + "api-key-and-credential-headers", + "proxy-forwarded-anthropic-beta", + "proxy-forwarded-anthropic-sdk-headers", + "proxy-forwarded-openai-organization", + "proxy-forwarded-anthropic-oauth-token", + "proxy-forwarded-provider-auth-header", + "proxy-forwarded-gateway-key", + "proxy-forwarding-nothing", + ], +) +async def test_caller_credential_fields_name_every_field_that_carries_a_caller_header_or_credential( + sent: dict[str, object], + request_headers: dict[str, str], + general_settings: dict[str, object], + expected: tuple[str, ...], +) -> None: + updated: Final = await _request_data_added_by_the_proxy(sent, request_headers, general_settings) + + assert updated["proxy_server_request"]["caller_credential_fields"] == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_headers,expected", + [({}, ()), ({"x-caller-trace": "trace-1"}, ("headers",))], + ids=["user-information-only", "user-information-and-caller-header"], +) +async def test_caller_credential_fields_ignore_the_user_information_the_proxy_adds_to_forwarded_headers( + monkeypatch: pytest.MonkeyPatch, request_headers: dict[str, str], expected: tuple[str, ...] +) -> None: + monkeypatch.setattr(litellm, "add_user_information_to_llm_headers", True) + + updated: Final = await _request_data_added_by_the_proxy({}, request_headers, FORWARD_PROVIDER_AUTH_HEADERS) + + assert updated["headers"]["x-litellm-user_api_key_user_id"] == "test-user" + assert updated["proxy_server_request"]["caller_credential_fields"] == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "sent,request_headers,general_settings,expected", + [ + ({"headers": {"Anthropic-Version": "2023-06-01"}}, {}, {}, ("headers",)), + ( + { + "provider_specific_header": { + "custom_llm_provider": "anthropic", + "extra_headers": {"anthropic-version": "2023-06-01"}, + } + }, + {}, + {}, + ("provider_specific_header",), + ), + ({"organization": "org-caller"}, {}, {}, ()), + ({"extra_headers": {"API-Key": "caller-azure-key"}}, {}, {}, ()), + ( + {"api_key": "sk-caller", "headers": {"Ocp-Apim-Subscription-Key": "caller"}}, + {}, + {}, + ("api_key", "headers"), + ), + ({}, {"openai-organization": "org-caller"}, {"forward_openai_org_id": True}, ()), + ( + {}, + {"authorization": "Bearer sk-virtual", "x-goog-api-key": "AIza-caller"}, + FORWARD_PROVIDER_AUTH_HEADERS, + (), + ), + ], + ids=[ + "anthropic-version-in-headers", + "anthropic-version-in-provider-specific-header", + "organization", + "api-key-in-extra-headers", + "api-key-and-credential-headers", + "proxy-forwarded-openai-organization", + "proxy-forwarded-provider-auth-header", + ], +) +async def test_credential_fields_name_the_stripped_fields_the_caller_sent( + sent: dict[str, object], + request_headers: dict[str, str], + general_settings: dict[str, object], + expected: tuple[str, ...], +) -> None: + updated: Final = await _request_data_added_by_the_proxy(sent, request_headers, general_settings) + + assert updated["proxy_server_request"]["credential_fields"] == expected + + +async def _request_data_added_by_the_proxy( + sent: dict[str, object], request_headers: dict[str, str], general_settings: dict[str, object] +) -> dict: + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + request_mock: Final = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.url.path = "/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"content-type": "application/json", **request_headers} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + data: Final = {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hello"}], **sent} + + return await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", user_id="test-user"), + proxy_config=MagicMock(), + general_settings=dict(general_settings), + version="test-version", + ) + + def test_initial_snapshot_refresh_clears_a_previous_guardrail_checkpoint() -> None: from litellm.integrations.shadow_eval_logger import GuardrailRequestSnapshot from litellm.litellm_core_utils.litellm_logging import Logging @@ -9208,6 +9425,97 @@ def test_default_team_settings_bool_turn_off_message_logging_redacts(): ) +def _fake_chatgpt_oauth_token(account_id: str) -> str: + header = base64.urlsafe_b64encode(b'{"alg":"RS256"}').rstrip(b"=").decode() + payload = ( + base64.urlsafe_b64encode( + json.dumps({"https://api.openai.com/auth": {"chatgpt_account_id": account_id}, "exp": 4102444800}).encode() + ) + .rstrip(b"=") + .decode() + ) + return f"{header}.{payload}.fake-signature" + + +CHATGPT_OAUTH_TOKEN = f"Bearer {_fake_chatgpt_oauth_token('acct-alice')}" +CHATGPT_ACCOUNT_ID_HEADER_NAME = "ChatGPT-Account-Id" +PLATFORM_API_KEY_BEARER = "Bearer sk-proj-fake-platform-key-0123456789" +CHATGPT_LEAK_TARGET_PROVIDERS = ["openai", "azure", "anthropic", "bedrock", "vertex_ai"] + + +def _codex_client_headers( + authorization_header_name: str = "authorization", account_id: str | None = "acct-alice" +) -> dict: + return { + "content-type": "application/json", + "originator": "codex_cli_rs", + authorization_header_name: CHATGPT_OAUTH_TOKEN, + **({CHATGPT_ACCOUNT_ID_HEADER_NAME: account_id} if account_id is not None else {}), + } + + +def test_clean_headers_keeps_chatgpt_oauth_bearer_when_gateway_key_is_separate(): + result = clean_headers( + headers=Headers({**_codex_client_headers(), "x-litellm-api-key": "sk-proxy-virtual-key"}), + litellm_key_header_name="x-litellm-api-key", + forward_llm_provider_auth_headers=False, + authenticated_with_header="x-litellm-api-key", + ) + + assert result["authorization"] == CHATGPT_OAUTH_TOKEN + assert "x-litellm-api-key" not in result + + +def test_clean_headers_drops_bearer_that_authenticated_to_the_gateway(): + result = clean_headers( + headers=Headers({"authorization": CHATGPT_OAUTH_TOKEN, "content-type": "application/json"}), + litellm_key_header_name=None, + forward_llm_provider_auth_headers=False, + authenticated_with_header="authorization", + ) + + assert _authorization_values(result) == [] + + +@pytest.mark.parametrize("bearer", [PLATFORM_API_KEY_BEARER, "Bearer not.a-jwt", "Bearer a.bm90LWpzb24.c"]) +def test_clean_headers_drops_non_chatgpt_bearers(bearer): + result = clean_headers( + headers=Headers({"authorization": bearer, "x-litellm-api-key": "sk-proxy-virtual-key"}), + litellm_key_header_name="x-litellm-api-key", + forward_llm_provider_auth_headers=False, + authenticated_with_header="x-litellm-api-key", + ) + + assert _authorization_values(result) == [] + + +@pytest.mark.parametrize("authorization_header_name", AUTHORIZATION_HEADER_CASINGS) +def test_chatgpt_oauth_credential_and_account_reach_chatgpt_provider(authorization_header_name): + forwarded = _headers_forwarded_to(_codex_client_headers(authorization_header_name), "chatgpt") + + assert _authorization_values(forwarded) == [CHATGPT_OAUTH_TOKEN] + assert forwarded[CHATGPT_ACCOUNT_ID_HEADER_NAME] == "acct-alice" + + +@pytest.mark.parametrize("custom_llm_provider", CHATGPT_LEAK_TARGET_PROVIDERS) +def test_chatgpt_oauth_credential_is_never_forwarded_to_other_providers(custom_llm_provider): + forwarded = _headers_forwarded_to(_codex_client_headers(), custom_llm_provider) + + assert _authorization_values(forwarded) == [] + assert CHATGPT_OAUTH_TOKEN not in forwarded.values() + assert CHATGPT_ACCOUNT_ID_HEADER_NAME not in forwarded + + +def test_chatgpt_account_header_is_not_forwarded_without_chatgpt_oauth_bearer(): + headers = {**_codex_client_headers(), "authorization": PLATFORM_API_KEY_BEARER} + + assert _headers_forwarded_to(headers, "chatgpt") == {} + + +def test_anthropic_oauth_bearer_does_not_reach_chatgpt_provider(): + assert _authorization_values(_headers_forwarded_to(_client_headers(), "chatgpt")) == [] + + def test_add_user_api_key_auth_to_request_metadata_attributes_a_cli_session_to_its_alias(): data = {"model": "gpt-5.4-nano", "messages": [{"role": "user", "content": "hi"}], "litellm_metadata": {}} session = UserAPIKeyAuth( diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 14196d4a1ac..745488bf93f 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -15207,21 +15207,23 @@ async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_coun assert_loop_stayed_free(took, lags) -async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeypatch): - from litellm.rust_bridge._native import Tokenizer +async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeypatch: pytest.MonkeyPatch) -> None: + from queue import SimpleQueue + from threading import Thread, current_thread + + from tokenizers import Tokenizer from litellm import Router - from tests.unit.litellm_core_utils.event_loop_lag import assert_loop_stayed_free, timed_with_loop_lags - claude_tokenizer: Final = litellm.utils.select_tokenizer("claude-fable-5")["tokenizer"] + claude_tokenizer: Final = Tokenizer.from_str(litellm.utils.claude_json_str) + loop_thread: Final = current_thread() + loader_threads: Final[SimpleQueue[Thread]] = SimpleQueue() - class SlowHubTokenizer: - @staticmethod - def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> Tokenizer: - time.sleep(0.3) - return claude_tokenizer + def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> Tokenizer: + loader_threads.put(current_thread()) + return claude_tokenizer - monkeypatch.setattr("litellm.rust_bridge.tokenizer.from_pretrained", SlowHubTokenizer.from_pretrained) + monkeypatch.setattr("tokenizers.Tokenizer", types.SimpleNamespace(from_pretrained=from_pretrained)) monkeypatch.setattr( "litellm.proxy.proxy_server.llm_router", Router( @@ -15237,15 +15239,16 @@ async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeyp ), ) - response, took, lags = await timed_with_loop_lags( - lambda: proxy_server_module.token_counter( + litellm.utils._select_custom_tokenizer_helper.cache_clear() + try: + response: Final = await proxy_server_module.token_counter( TokenCountRequest(model="self-hosted", prompt="count me off the loop") ) - ) - - assert response.tokenizer_type == "huggingface_tokenizer" - assert response.total_tokens > 0 - assert_loop_stayed_free(took, lags) + assert loader_threads.get_nowait() is not loop_thread + assert response.tokenizer_type == "huggingface_tokenizer" + assert response.total_tokens > 0 + finally: + litellm.utils._select_custom_tokenizer_helper.cache_clear() async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revision_and_token(monkeypatch): @@ -15584,4 +15587,3 @@ async def test_spend_capture_rate_check_job_clears_the_gauge_once_the_setting_is call(api_provider="openai", capture_rate=0.97), call(api_provider="openai", capture_rate=None), ] - diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 97a39f64001..0863abd1f1e 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -933,6 +933,42 @@ class TestResponsesInputToChatMessages: ) == [user_message] +class TestMergeProviderSpecificHeaders: + def test_scoped_headers_merge_only_for_matching_provider(self): + scoped = {"custom_llm_provider": "chatgpt", "extra_headers": {"Authorization": "Bearer client-token"}} + + merged = ResponsesAPIRequestUtils.merge_provider_specific_headers( + extra_headers={"originator": "codex_cli_rs"}, + provider_specific_header=scoped, + custom_llm_provider="chatgpt", + ) + untouched = ResponsesAPIRequestUtils.merge_provider_specific_headers( + extra_headers={"originator": "codex_cli_rs"}, + provider_specific_header=scoped, + custom_llm_provider="openai", + ) + + assert merged == {"originator": "codex_cli_rs", "Authorization": "Bearer client-token"} + assert untouched == {"originator": "codex_cli_rs"} + + def test_sequence_of_scoped_entries_each_keep_their_own_scope(self): + entries = [ + {"custom_llm_provider": "anthropic", "extra_headers": {"Authorization": "Bearer anthropic-token"}}, + {"custom_llm_provider": "chatgpt", "extra_headers": {"Authorization": "Bearer chatgpt-token"}}, + ] + + merged = ResponsesAPIRequestUtils.merge_provider_specific_headers( + extra_headers=None, provider_specific_header=entries, custom_llm_provider="chatgpt" + ) + + assert merged == {"Authorization": "Bearer chatgpt-token"} + + def test_no_scoped_headers_returns_extra_headers_unchanged(self): + assert ResponsesAPIRequestUtils.merge_provider_specific_headers( + extra_headers=None, provider_specific_header=None, custom_llm_provider="chatgpt" + ) is None + + def test_responses_object_bills_an_explicit_cache_hit_at_the_explicit_rate(): model = "dashscope/qwen-responses-explicit-cache-test" previous_row = litellm.model_cost.get(model) diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index f956bb66e6c..a46f415e10a 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -112,6 +112,13 @@ def _heuristic_v2_artifact() -> TrainedTierArtifact: ) +def _answer_deployments_named(router: MagicMock) -> None: + """Make a mock router answer `deployments_named` from its own `model_list`, the way Router answers from its index.""" + router.deployments_named.side_effect = lambda name: ( + tuple(deployment for deployment in router.model_list if deployment["model_name"] == name) or None + ) + + @pytest.fixture def mock_router_instance(): """Create a mock LiteLLM Router instance.""" @@ -1065,6 +1072,11 @@ class TestCustomDimensions: "create table widgets (id integer); ALTER TABLE widgets ADD label text;", id="regex", ), + pytest.param( + {"patterns": [r"\b(?Pcreate|alter)\s{1,4}?table\b"]}, + "CREATE TABLE widgets (id integer)", + id="named-group-alternation-lazy-repeat", + ), ], ) def test_custom_dimension_changes_only_matching_requests( @@ -3176,7 +3188,7 @@ class TestCapabilityClassifier: {"model_name": "efficient-model", "litellm_params": {"input_cost_per_token": 0.000001}}, {"model_name": "capable-model", "litellm_params": {"input_cost_per_token": 0.00001}}, ] - mock_router_instance.model_name_to_deployment_indices = {"efficient-model": [0], "capable-model": [1]} + _answer_deployments_named(mock_router_instance) router: Final = self._router( mock_router_instance, adaptive=True, @@ -5665,7 +5677,7 @@ class TestAdaptiveSoftFloors: "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, }, ] - router.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + _answer_deployments_named(router) return router @pytest.fixture @@ -6824,7 +6836,7 @@ class TestTierModelAffinity: ) -> ComplexityRouter: mock_router_instance.cache = DualCache() mock_router_instance.model_list = [] - mock_router_instance.model_name_to_deployment_indices = {} + _answer_deployments_named(mock_router_instance) return ComplexityRouter( model_name="affinity-router", litellm_router_instance=mock_router_instance, @@ -7542,7 +7554,7 @@ class TestSessionAffinity: "model_info": {}, }, ] - mock_router_instance.model_name_to_deployment_indices = {"cheap": [0]} + _answer_deployments_named(mock_router_instance) router = ComplexityRouter( model_name="hybrid", litellm_router_instance=mock_router_instance, @@ -11346,7 +11358,7 @@ class TestClientHousekeepingCalls: "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, }, ] - adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + _answer_deployments_named(adaptive_instance) router = ComplexityRouter( model_name="hybrid", litellm_router_instance=adaptive_instance, @@ -13437,7 +13449,7 @@ class TestPlanModeTierFloor: "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}, }, ] - adaptive_instance.model_name_to_deployment_indices = {"cheap": [0], "premium": [1]} + _answer_deployments_named(adaptive_instance) router = ComplexityRouter( model_name="hybrid", litellm_router_instance=adaptive_instance, @@ -14936,7 +14948,7 @@ class TestModalityRouting: elif path == "adaptive_pick_rewritten": config["adaptive"] = True mock_router_instance.model_list = [] - mock_router_instance.model_name_to_deployment_indices = {} + _answer_deployments_named(mock_router_instance) router = self._router(mock_router_instance, config, vision) result = await router.async_pre_routing_hook(model="m", request_kwargs=request_kwargs, messages=messages) assert result.model == expected_model diff --git a/tests/unit/router_strategy/test_savings_baseline.py b/tests/unit/router_strategy/test_savings_baseline.py index 5efc73d2dcb..4241a541202 100644 --- a/tests/unit/router_strategy/test_savings_baseline.py +++ b/tests/unit/router_strategy/test_savings_baseline.py @@ -107,8 +107,7 @@ class TestResolveBaseline: served; a dashboard counterfactual must not be able to take routing down.""" class Exploding: - @property - def model_name_to_deployment_indices(self): + def deployments_named(self, model_name): raise RuntimeError("router is mid-reload") assert resolve_baseline(Exploding(), ["anything"]) is None diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index d634e239ea9..d0564812c48 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -203,7 +203,7 @@ class TestNoneLevelPolarity: def test_azure_advertisement_matches_the_azure_request_gate(self, model_key): """AzureOpenAIGPT5Config raises UnsupportedParamsError on reasoning_effort='none' for models it does not flag, so advertising the level there would offer routing a 400.""" - from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config + from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config from litellm.utils import get_model_info_helper model_info = dict(get_model_info_helper(model=model_key.split("/", 1)[1], custom_llm_provider="azure")) diff --git a/tests/unit/secret_managers/test_google_secret_manager.py b/tests/unit/secret_managers/test_google_secret_manager.py index 2051be08f7c..7effde4fb30 100644 --- a/tests/unit/secret_managers/test_google_secret_manager.py +++ b/tests/unit/secret_managers/test_google_secret_manager.py @@ -1,6 +1,7 @@ import base64 import binascii import json +from collections import OrderedDict from dataclasses import dataclass from typing import Final @@ -120,7 +121,7 @@ def manager(monkeypatch: pytest.MonkeyPatch) -> GoogleSecretManager: monkeypatch.setattr(litellm, "secret_manager_client", None) monkeypatch.setattr(litellm, "_key_management_system", None) monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings()) - monkeypatch.setattr(litellm.vertex_chat_completion, "_credentials_project_mapping", {}) + monkeypatch.setattr(litellm.vertex_chat_completion, "_credentials_project_mapping", OrderedDict()) monkeypatch.setattr(google.auth, "default", _default_credentials) monkeypatch.setenv("GOOGLE_SECRET_MANAGER_PROJECT_ID", PROJECT_ID) monkeypatch.delenv("GOOGLE_SECRET_MANAGER_ALWAYS_READ_SECRET_MANAGER", raising=False) diff --git a/tests/unit/test_auto_update_price_and_context_window_file.py b/tests/unit/test_auto_update_price_and_context_window_file.py deleted file mode 100644 index 435747e9a09..00000000000 --- a/tests/unit/test_auto_update_price_and_context_window_file.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Unit tests for the Friendli transform in -`.github/scripts/auto_update_price_and_context_window_file.py`.""" - -from __future__ import annotations - -import importlib.util -import sys -from pathlib import Path - -import pytest - -SCRIPT_PATH = ( - Path(__file__).resolve().parents[2] - / ".github" - / "scripts" - / "auto_update_price_and_context_window_file.py" -) - - -@pytest.fixture(scope="module") -def sync_module(): - spec = importlib.util.spec_from_file_location( - "auto_update_price_and_context_window_file", SCRIPT_PATH - ) - assert spec and spec.loader, f"Could not load spec for {SCRIPT_PATH}" - module = importlib.util.module_from_spec(spec) - sys.modules["auto_update_price_and_context_window_file"] = module - spec.loader.exec_module(module) - return module - - -def _reasoning_model(**overrides: object) -> dict: - model = { - "id": "zai-org/GLM-Test", - "base_model": "zhipuai/glm-test", - "context_length": 1048576, - "max_completion_tokens": 131072, - "pricing": {"input": "0.00000015", "output": "0.0000005", "input_cache_read": "0.00000003"}, - "reasoning": True, - "reasoning_options": [{"type": "effort", "values": ["max", "high", "low"]}], - "functionality": { - "tool_call": True, - "parallel_tool_call": True, - "structured_output": True, - "system_messages": True, - "tool_choice": True, - }, - "input_modalities": ["text", "image", "video"], - "mode": "chat", - } - model.update(overrides) - return model - - -def test_transform_emits_declared_effort_levels_in_canonical_order(sync_module): - entry = sync_module.transform_friendli_data([_reasoning_model()], {})[ - "friendliai/zai-org/GLM-Test" - ] - assert entry["supports_reasoning"] is True - assert entry["reasoning_effort_levels"] == ["low", "high", "max"] - assert not any(k.endswith("_reasoning_effort") for k in entry) - - -def test_transform_reasoning_model_without_effort_options_declares_empty_levels(sync_module): - model = _reasoning_model(reasoning_options=[{"type": "budget_tokens", "values": []}]) - entry = sync_module.transform_friendli_data([model], {})["friendliai/zai-org/GLM-Test"] - assert entry["reasoning_effort_levels"] == [] - - -def test_transform_non_reasoning_model_declares_no_levels(sync_module): - model = _reasoning_model(reasoning=False, reasoning_options=[]) - entry = sync_module.transform_friendli_data([model], {})["friendliai/zai-org/GLM-Test"] - assert entry["supports_reasoning"] is False - assert "reasoning_effort_levels" not in entry - - -def test_transform_max_tokens_mirrors_output_cap_not_context(sync_module): - entry = sync_module.transform_friendli_data([_reasoning_model()], {})[ - "friendliai/zai-org/GLM-Test" - ] - assert entry["max_input_tokens"] == 1048576 - assert entry["max_output_tokens"] == 131072 - assert entry["max_tokens"] == entry["max_output_tokens"] - - -def test_transform_prompt_caching_follows_cache_pricing(sync_module): - cached = sync_module.transform_friendli_data([_reasoning_model()], {})[ - "friendliai/zai-org/GLM-Test" - ] - assert cached["supports_prompt_caching"] is True - assert cached["cache_read_input_token_cost"] == 3e-08 - - uncached_model = _reasoning_model(pricing={"input": "0.00000014", "output": "0.0000004"}) - uncached = sync_module.transform_friendli_data([uncached_model], {})[ - "friendliai/zai-org/GLM-Test" - ] - assert uncached["supports_prompt_caching"] is False - assert "cache_read_input_token_cost" not in uncached - - -def test_transform_modalities_set_vision_image_and_video_flags(sync_module): - entry = sync_module.transform_friendli_data([_reasoning_model()], {})[ - "friendliai/zai-org/GLM-Test" - ] - assert entry["supports_vision"] is True - assert entry["supports_image_input"] is True - assert entry["supports_video_input"] is True - - text_only = _reasoning_model(input_modalities=["text"]) - entry_text = sync_module.transform_friendli_data([text_only], {})[ - "friendliai/zai-org/GLM-Test" - ] - assert entry_text["supports_vision"] is False - assert entry_text["supports_image_input"] is False - assert entry_text["supports_video_input"] is False - - -def test_transform_skips_rows_without_valid_token_prices_so_priced_local_entries_survive(sync_module): - local = { - "friendliai/zai-org/GLM-Test": { - "litellm_provider": "friendliai", - "input_cost_per_token": 1.5e-07, - "output_cost_per_token": 5e-07, - } - } - unpriced_rows = [ - _reasoning_model(pricing={}), - _reasoning_model(pricing=None), - _reasoning_model(pricing={"input": "0.00000015"}), - _reasoning_model(pricing={"output": "0.0000005"}), - _reasoning_model(pricing={"input": "not-a-number", "output": "0.0000005"}), - _reasoning_model(pricing={"input": "-0.00000015", "output": "0.0000005"}), - _reasoning_model(pricing={"input": "inf", "output": "0.0000005"}), - _reasoning_model(pricing={"input": "nan", "output": "0.0000005"}), - ] - remote = sync_module.transform_friendli_data(unpriced_rows, local) - assert remote == {} - sync_module.sync_local_data_with_remote(local, remote, replace_keys=frozenset(remote)) - assert local["friendliai/zai-org/GLM-Test"]["input_cost_per_token"] == 1.5e-07 - assert local["friendliai/zai-org/GLM-Test"]["output_cost_per_token"] == 5e-07 - - -def test_transform_keeps_zero_priced_rows(sync_module): - free_model = _reasoning_model(pricing={"input": "0", "output": "0"}) - entry = sync_module.transform_friendli_data([free_model], {})["friendliai/zai-org/GLM-Test"] - assert entry["input_cost_per_token"] == 0.0 - assert entry["output_cost_per_token"] == 0.0 - - -def test_transforms_survive_failed_fetch(sync_module): - assert sync_module.transform_friendli_data(None, {}) == {} - assert sync_module.transform_friendli_data([], {}) == {} - assert sync_module.transform_openrouter_data(None) == {} - assert sync_module.transform_vercel_ai_gateway_data(None) == {} - - -def test_vercel_transform_skips_rows_without_token_pricing_or_limits(sync_module): - rows = [ - { - "id": "wan-video", - "pricing": {"video_duration_pricing": [{"resolution": "720p", "cost_per_second": "0.1"}]}, - }, - { - "id": "qwen3-embedding", - "context_window": 32768, - "max_tokens": 32768, - "pricing": {"input": "0.00000001"}, - }, - { - "id": "no-limits-chat", - "pricing": {"input": "0.000001", "output": "0.000002"}, - }, - { - "id": "good-chat", - "context_window": 128000, - "max_tokens": 8192, - "pricing": {"input": "0.000001", "output": "0.000002"}, - }, - ] - transformed = sync_module.transform_vercel_ai_gateway_data(rows) - assert list(transformed) == ["vercel_ai_gateway/good-chat"] - assert transformed["vercel_ai_gateway/good-chat"]["input_cost_per_token"] == 1e-06 - assert transformed["vercel_ai_gateway/good-chat"]["output_cost_per_token"] == 2e-06 - - -def test_sync_replaces_friendli_entries_so_dropped_cache_pricing_does_not_survive(sync_module): - local = { - "friendliai/zai-org/GLM-Test": { - "litellm_provider": "friendliai", - "cache_read_input_token_cost": 3e-08, - "supports_prompt_caching": True, - } - } - uncached_model = _reasoning_model(pricing={"input": "0.00000014", "output": "0.0000004"}) - remote = sync_module.transform_friendli_data([uncached_model], local) - sync_module.sync_local_data_with_remote(local, remote, replace_keys=frozenset(remote)) - synced = local["friendliai/zai-org/GLM-Test"] - assert "cache_read_input_token_cost" not in synced - assert synced["supports_prompt_caching"] is False - - -def test_sync_still_merges_entries_outside_replace_keys(sync_module): - local = {"openrouter/some-model": {"input_cost_per_token": 1e-06, "supports_vision": True}} - remote = {"openrouter/some-model": {"input_cost_per_token": 2e-06}} - sync_module.sync_local_data_with_remote(local, remote) - assert local["openrouter/some-model"] == {"input_cost_per_token": 2e-06, "supports_vision": True} - - -def test_transform_inherits_allowlisted_keys_from_base_model_entry(sync_module): - local = { - "zhipuai/glm-test": { - "supports_pdf_input": True, - "supports_assistant_prefill": True, - "input_cost_per_token": 9e-06, - } - } - entry = sync_module.transform_friendli_data([_reasoning_model()], local)[ - "friendliai/zai-org/GLM-Test" - ] - assert entry["supports_pdf_input"] is True - assert entry["supports_assistant_prefill"] is True - assert entry["input_cost_per_token"] == 1.5e-07 diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index c0abb18ca8c..1f84be7710f 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -12,7 +12,7 @@ from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter import litellm from litellm.litellm_core_utils.fallback_generalizations import match_capability_generalizations -from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name +from litellm.llms.openai.chat.gpt_reasoning_transformation import is_gpt_reasoning_series_name from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts from litellm.types.utils import SearchFreeRequests @@ -27,13 +27,29 @@ def prices() -> dict: return json.loads(PRICES_PATH.read_text()) +@pytest.mark.parametrize("model", ("azure/gpt-audio-1.5-2026-02-23", "azure/gpt-audio-mini-2025-10-06")) +def test_azure_audio_snapshot_capabilities_match_declared_modalities(model: str, local_model_cost_map: None) -> None: + catalog: Final = TypeAdapter(dict[str, JsonValue]).validate_json(PRICES_PATH.read_bytes()) + backup: Final = TypeAdapter(dict[str, JsonValue]).validate_json(BACKUP_PRICES_PATH.read_bytes()) + entry: Final = catalog[model] + assert isinstance(entry, dict) + assert entry == backup[model] + input_modalities: Final = entry["supported_modalities"] + output_modalities: Final = entry["supported_output_modalities"] + assert isinstance(input_modalities, list) + assert isinstance(output_modalities, list) + expected: Final = ("audio" in input_modalities, "audio" in output_modalities) + info: Final = litellm.get_model_info(model) + assert (info["supports_audio_input"], info["supports_audio_output"]) == expected + assert (litellm.supports_audio_input(model), litellm.supports_audio_output(model)) == expected + + class _CopilotEndpointRow(BaseModel): model_config = ConfigDict(frozen=True, extra="ignore") mode: str max_input_tokens: int max_output_tokens: int - max_tokens: int supported_endpoints: tuple[str, ...] @@ -79,18 +95,22 @@ def _anthropic_counterpart(key: str) -> str: @pytest.mark.parametrize(("path",), [(PRICES_PATH,), (BACKUP_PRICES_PATH,)], ids=("main", "backup")) -def test_github_copilot_rows_derive_mode_and_max_tokens_from_their_endpoints(path: Path) -> None: +def test_github_copilot_rows_derive_mode_from_their_endpoints(path: Path) -> None: rows: Final[Mapping[str, _CopilotEndpointRow]] = _copilot_endpoint_rows(path) assert rows, f"No GitHub Copilot endpoint rows found in {path}" - assert {key: (row.mode, row.max_tokens) for key, row in rows.items()} == { - key: ( - "chat" if "/v1/chat/completions" in row.supported_endpoints else "responses", - row.max_output_tokens, - ) - for key, row in rows.items() + assert {key: row.mode for key, row in rows.items()} == { + key: "chat" if "/v1/chat/completions" in row.supported_endpoints else "responses" for key, row in rows.items() } +def _rows_carrying_legacy_max_tokens(prices: Mapping[str, Mapping[str, object]]) -> tuple[str, ...]: + return tuple(key for key, row in prices.items() if "max_tokens" in row) + + +def test_no_row_carries_the_retired_max_tokens_field(prices: dict) -> None: + assert _rows_carrying_legacy_max_tokens(prices) == () + + def test_github_copilot_backup_rows_match_the_main_catalog() -> None: assert _github_copilot_catalog_rows(PRICES_PATH) == _github_copilot_catalog_rows(BACKUP_PRICES_PATH) @@ -584,7 +604,6 @@ def test_tencent_tokenhub_glm_rows_match_published_metadata(path: Path): or entry.get("litellm_provider") != "tencent" or entry.get("max_input_tokens") != 1000000 or entry.get("max_output_tokens") != 128000 - or entry.get("max_tokens") != 128000 or entry.get("input_cost_per_token_cache_hit") != entry.get("cache_read_input_token_cost") ) assert actual == TENCENT_TOKENHUB_GLM_ROWS @@ -602,7 +621,7 @@ def test_tencent_tokenhub_kimi_rows_match_published_metadata(path: Path): entry.get("input_cost_per_token"), entry.get("output_cost_per_token"), entry.get("cache_read_input_token_cost"), - entry.get("max_tokens"), + entry.get("max_output_tokens"), entry.get("supports_vision", False), entry.get("supports_video_input", False), ) @@ -776,7 +795,6 @@ def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(pa assert entry["mode"] == "chat" assert entry["max_input_tokens"] == 1024000 assert entry["max_output_tokens"] == 256000 - assert entry["max_tokens"] == 256000 # Volcengine Ark pricing https://www.volcengine.com/docs/82379/1544106 checked 2026-10-20 assert entry["input_cost_per_token"] == pytest.approx(6.0 / 6.9565 / 1_000_000, rel=1e-4) assert entry["output_cost_per_token"] == pytest.approx(30.0 / 6.9565 / 1_000_000, rel=1e-4) diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index e73b9950594..dcd18a5c75e 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -2233,7 +2233,7 @@ LLMS_ALIAS_CASES: Final = ( ("litellm.llms.oci.chat.generic", "", "_normalize_oci_finish_reason", "normalize_oci_finish_reason", False), ("litellm.llms.oci.chat.generic", "", "_synthesize_oci_tool_call_id", "synthesize_oci_tool_call_id", False), ("litellm.llms.ollama.common_utils", "", "_convert_image", "convert_image", False), - ("litellm.llms.openai.chat.gpt_5_transformation", "", "_get_effort_level", "get_effort_level", False), + ("litellm.llms.openai.chat.gpt_reasoning_transformation", "", "_get_effort_level", "get_effort_level", False), ("litellm.llms.openai.completion.utils", "", "_transform_prompt", "transform_prompt", False), ( "litellm.llms.openai.cost_calculation", @@ -4867,7 +4867,7 @@ LLMS_FORWARDER_CASES: Final = ( False, ), ( - "litellm.llms.openai.chat.gpt_5_transformation", + "litellm.llms.openai.chat.gpt_reasoning_transformation", "OpenAIGPT5Config", "_supports_reasoning_effort_level", "supports_reasoning_effort_level", diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 568a05e8ff4..3004f83b1df 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -5,6 +5,7 @@ import functools import gc import itertools import json +import linecache import logging import os import sys @@ -13,8 +14,8 @@ import warnings from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator, Mapping from datetime import datetime, timedelta, timezone from pathlib import Path -from types import MappingProxyType, SimpleNamespace -from typing import TYPE_CHECKING, Final, Literal, cast +from types import CodeType, FrameType, MappingProxyType, SimpleNamespace +from typing import TYPE_CHECKING, Final, Literal, TypeVar, cast from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx @@ -60,7 +61,10 @@ from litellm.router import ( _without_line_breaks, Span, ) +from litellm.proxy.response_polling.polling_handler import should_use_polling_for_request from litellm.router_strategy import simple_shuffle +from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter +from litellm.router_strategy.savings_baseline import Baseline, _models_in from litellm.router_utils.client_initalization_utils import MaxParallelRequestsLimit from litellm.router_utils.cooldown_handlers import ( _async_get_cooldown_deployments, @@ -279,6 +283,15 @@ def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata(): assert kwargs["litellm_metadata"] == {"baz": 123} +def test_router_model_group_llm_provider_resolves_chatgpt_without_login(tmp_path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + params: Final = LiteLLM_Params(model="chatgpt/gpt-5.4") + + assert Router._model_group_llm_provider(params) == ("gpt-5.4", "chatgpt") + assert Router._model_group_llm_provider(LiteLLM_Params(model="no-such-provider-model-xyz")) == ("", "") + assert not any(tmp_path.iterdir()) + + def test_router_with_model_info_and_model_group(): """ Test edge case where user specifies model_group in model_info @@ -8872,7 +8885,8 @@ def test_get_wildcard_deployment_usable_by_team_prefers_the_team_pattern(): pytest.param(False, "rwild-chat", "team-r", ("team-r-wildcard",), id="team-pattern-when-no-proxy-wide-match"), pytest.param(False, "rwild-chat", "team-b", (), id="another-teams-pattern-never-serves"), pytest.param(False, "rwild-chat", None, (), id="no-team-no-team-pattern"), - pytest.param(True, "rwild-chat", "team-r", ("catch-all",), id="proxy-wide-pattern-beats-the-team-pattern"), + pytest.param(True, "rwild-chat", "team-r", ("team-r-wildcard",), id="team-pattern-beats-the-proxy-wide-pattern"), + pytest.param(True, "rwild-chat", "team-b", ("catch-all",), id="another-team-falls-through-to-proxy-wide"), pytest.param(True, "ptu-chat", "team-r", ("ptu-wildcard",), id="only-the-most-specific-proxy-wide-pattern"), ], ) @@ -8911,6 +8925,39 @@ def test_wildcard_route_deployments_is_the_one_pattern_routing_serves_from( ) +@pytest.mark.parametrize( + ("team_id", "expected_id"), + [ + pytest.param("team-a", "team-a-byok", id="team-key-uses-its-own-byok-wildcard"), + pytest.param("team-b", "shared-openrouter", id="team-without-byok-uses-the-shared-wildcard"), + pytest.param(None, "shared-openrouter", id="key-without-team-uses-the-shared-wildcard"), + ], +) +def test_team_byok_wildcard_is_served_before_a_proxy_wide_wildcard_of_the_same_name( + team_id: str | None, expected_id: str +): + """Regression: a proxy-wide openrouter/* shadowed a team's own openrouter/* BYOK deployment for that team's keys.""" + router: Final = litellm.Router( + model_list=[ + { + "model_name": "model_name_team-a_internal", + "litellm_params": {"model": "openrouter/*", "api_key": "sk-team-a"}, + "model_info": {"id": "team-a-byok", "team_id": "team-a", "team_public_model_name": "openrouter/*"}, + }, + { + "model_name": "openrouter/*", + "litellm_params": {"model": "openrouter/*", "api_key": "sk-shared"}, + "model_info": {"id": "shared-openrouter"}, + }, + ] + ) + requested: Final = "openrouter/openai/gpt-5-nano" + + served: Final = router.wildcard_route_deployments(requested, team_id) + + assert tuple(Deployment.model_validate(d).model_info.id for d in served) == (expected_id,) + + def test_get_deployment_credentials_with_provider_aws_bedrock_runtime_endpoint(): """ Test that get_deployment_credentials_with_provider correctly copies @@ -30004,6 +30051,319 @@ async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_s assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens + fallback_tokens +_SYNC_SECONDS: Final = 10 +_ROUTER_FILE: Final = Router.__init__.__code__.co_filename +_INDEX_LOCK_LINES: Final = frozenset( + number + for number, line in enumerate(linecache.getlines(_ROUTER_FILE), start=1) + if "with self._deployment_index_lock" in line +) + +_Read = TypeVar("_Read") + + +def _concurrent_delete_deployment(index: int) -> dict[str, object]: + return { + "model_name": f"model-{index}", + "litellm_params": {"model": f"openai/gpt-{index}", "api_key": "k"}, + "model_info": { + "id": f"deployment-{index}", + "team_id": "team-1", + "team_public_model_name": "public-shared" if index >= 2 else f"public-{index}", + }, + } + + +def _concurrent_delete_router() -> Router: + return Router(model_list=[_concurrent_delete_deployment(index) for index in range(4)]) + + +def _read_during_held_write( + read: Callable[[], _Read], + write: Callable[[], object], + event: Literal["call", "c_call"] = "call", + write_after: CodeType | None = None, +) -> _Read: + """Run `read` on its own thread while `write` waits at its first `event` in the re-index, after the list shrank and + before the maps caught up. With `write_after`, the write starts when `read` returns from that function. The write + resumes once `read` returns or reaches the router's index lock, so a read that skips the lock meets the half-updated + list and a read that takes the lock waits for the write""" + reindex: Final = Router._update_deployment_indices_after_removal.__code__ + held: Final = threading.Event() + release: Final = threading.Event() + settled: Final = threading.Event() + released_in_time: Final[list[bool]] = [] + outcome: Final[list[_Read]] = [] + failure: Final[list[BaseException]] = [] + + def hold_in_reindex(frame: FrameType, observed: str, _arg: object) -> None: + if observed == event and frame.f_code is reindex and not held.is_set(): + held.set() + released_in_time.append(release.wait(timeout=_SYNC_SECONDS)) + + def run_write() -> None: + previous: Final = sys.getprofile() + sys.setprofile(hold_in_reindex) + try: + write() + finally: + sys.setprofile(previous) + + writer: Final = threading.Thread(target=run_write) + + def start_write() -> None: + writer.start() + assert held.wait(timeout=_SYNC_SECONDS), "the write never reached re-indexing" + + def watch_read(frame: FrameType, observed: str, _arg: object) -> Callable[..., object] | None: + if frame.f_code.co_filename != _ROUTER_FILE: + return None + if observed == "return" and frame.f_code is write_after and writer.ident is None: + start_write() + elif observed == "line" and held.is_set() and frame.f_lineno in _INDEX_LOCK_LINES: + settled.set() + return watch_read + + def run_read() -> None: + previous: Final = sys.gettrace() + sys.settrace(watch_read) + try: + outcome.append(read()) + except BaseException as error: + failure.append(error) + finally: + sys.settrace(previous) + settled.set() + + reader: Final = threading.Thread(target=run_read) + try: + if write_after is None: + start_write() + reader.start() + assert settled.wait(timeout=_SYNC_SECONDS), "the read neither returned nor reached the index lock" + finally: + release.set() + if reader.ident is not None: + reader.join() + if writer.ident is not None: + writer.join() + assert released_in_time == [True], "the write was not held for the whole read" + if failure: + raise failure[0] + return outcome[0] + + +def _deployment_ids(deployments: list[DeploymentTypedDict]) -> tuple[str, ...]: + return tuple(deployment["model_info"]["id"] for deployment in deployments) + + +@pytest.mark.parametrize( + ("read", "expected"), + [ + pytest.param( + lambda router: router.get_deployment("deployment-3").model_info.id, "deployment-3", id="get_deployment" + ), + pytest.param( + lambda router: router.get_model_info("deployment-3")["model_info"]["id"], + "deployment-3", + id="get_model_info", + ), + pytest.param( + lambda router: _deployment_ids(router.get_all_deployments("model-3")), + ("deployment-3",), + id="get_all_deployments", + ), + pytest.param( + lambda router: _deployment_ids(router.get_all_deployments("public-shared", team_id="team-1")), + ("deployment-2", "deployment-3"), + id="get_all_deployments-team", + ), + pytest.param( + lambda router: router.map_team_model("public-shared", "team-1"), "public-shared", id="map_team_model" + ), + pytest.param(lambda router: router.get_model_ids("model-3"), ["deployment-3"], id="get_model_ids-by-name"), + pytest.param( + lambda router: router.get_model_ids(), ["deployment-1", "deployment-2", "deployment-3"], id="get_model_ids" + ), + pytest.param( + lambda router: router.get_deployment_by_model_group_name("model-3").model_info.id, + "deployment-3", + id="get_deployment_by_model_group_name", + ), + pytest.param( + lambda router: router.get_model_listing_info("model-3").cost_map_keys, + ("openai/gpt-3",), + id="get_model_listing_info", + ), + pytest.param( + lambda router: router.team_model_has_alternatives("deployment-3"), True, id="team_model_has_alternatives" + ), + pytest.param( + lambda router: ( + router._get_model_group_deployment_usable_by_team( + model_group_name="model-3", team_id="team-1" + ).model_info.id + ), + "deployment-3", + id="model_group_deployment_usable_by_team", + ), + pytest.param( + lambda router: ( + router._get_team_public_name_deployment(model_id="public-shared", team_id="team-1").model_info.id + ), + "deployment-2", + id="team_public_name_deployment", + ), + pytest.param( + lambda router: _deployment_ids(router._team_deployments_across_teams("public-shared")), + ("deployment-2", "deployment-3"), + id="team_deployments_across_teams", + ), + pytest.param( + lambda router: _deployment_ids(list(router.deployments_named("model-3") or ())), + ("deployment-3",), + id="deployments_named", + ), + pytest.param( + lambda router: should_use_polling_for_request( + background_mode=True, + polling_via_cache_enabled=["openai"], + redis_cache=MagicMock(), + model="model-3", + llm_router=router, + ), + True, + id="response_polling_provider_lookup", + ), + pytest.param( + lambda router: _models_in(router, "model-3"), + (Baseline("openai/gpt-3", "deployment-3"),), + id="savings_baseline_candidates", + ), + ], +) +def test_lookup_during_a_concurrent_delete_returns_the_requested_deployment( + read: Callable[[Router], object], expected: object +) -> None: + router: Final = _concurrent_delete_router() + + observed: Final = _read_during_held_write(lambda: read(router), lambda: router.delete_deployment("deployment-0")) + + assert observed == expected + + +def test_lookup_during_an_upsert_returns_the_requested_deployment() -> None: + router: Final = _concurrent_delete_router() + rotated: Final = Deployment( + **{**_concurrent_delete_deployment(0), "litellm_params": {"model": "openai/gpt-0", "api_key": "rotated"}} + ) + + observed: Final = _read_during_held_write( + lambda: router.get_deployment("deployment-3").model_info.id, lambda: router.upsert_deployment(rotated) + ) + + assert observed == "deployment-3" + + +def test_add_during_a_concurrent_delete_indexes_the_added_deployment() -> None: + router: Final = _concurrent_delete_router() + + _read_during_held_write( + lambda: router.add_deployment(Deployment(**_concurrent_delete_deployment(4))), + lambda: router.delete_deployment("deployment-0"), + ) + + assert [router.get_deployment(f"deployment-{index}").model_info.id for index in range(1, 5)] == [ + f"deployment-{index}" for index in range(1, 5) + ] + + +def test_set_model_list_during_a_concurrent_delete_indexes_the_new_list() -> None: + router: Final = _concurrent_delete_router() + + _read_during_held_write( + lambda: router.set_model_list([_concurrent_delete_deployment(index) for index in range(10, 14)]), + lambda: router.delete_deployment("deployment-0"), + event="c_call", + ) + + assert router.get_model_ids() == [f"deployment-{index}" for index in range(10, 14)] + + +def test_team_model_has_alternatives_when_a_delete_starts_after_its_deployment_lookup() -> None: + router: Final = _concurrent_delete_router() + + has_alternatives: Final = _read_during_held_write( + lambda: router.team_model_has_alternatives("deployment-3"), + lambda: router.delete_deployment("deployment-0"), + write_after=Router.get_deployment.__code__, + ) + + assert has_alternatives is True + + +def test_init_adaptive_router_deployment_during_a_concurrent_delete_prices_the_listed_model() -> None: + priced: Final = { + **_concurrent_delete_deployment(3), + "litellm_params": {"model": "openai/gpt-3", "api_key": "k", "input_cost_per_token": 3e-06}, + } + router: Final = Router(model_list=[*(_concurrent_delete_deployment(index) for index in range(3)), priced]) + adaptive: Final = Deployment( + model_name="adaptive", + litellm_params=LiteLLM_Params( + model="auto_router/adaptive_router", adaptive_router_config={"available_models": ["model-3"]} + ), + model_info=ModelInfo(id="adaptive"), + ) + + _read_during_held_write( + lambda: router.init_adaptive_router_deployment(deployment=adaptive), + lambda: router.delete_deployment("deployment-0"), + ) + + assert router.adaptive_routers["adaptive"][0].strategy.model_to_cost == {"model-3": 3e-06} + + +def test_complexity_router_adaptive_pool_during_a_concurrent_delete_prices_the_listed_model() -> None: + priced: Final = { + **_concurrent_delete_deployment(3), + "litellm_params": {"model": "openai/gpt-3", "api_key": "k", "input_cost_per_token": 3e-06}, + } + router: Final = Router(model_list=[*(_concurrent_delete_deployment(index) for index in range(3)), priced]) + complexity: Final = ComplexityRouter( + model_name="hybrid", + litellm_router_instance=router, + complexity_router_config={"adaptive": True, "tiers": {"SIMPLE": ["model-3"]}}, + ) + + adaptive: Final = _read_during_held_write( + complexity._ensure_adaptive_router, lambda: router.delete_deployment("deployment-0") + ) + + assert adaptive is not None + assert adaptive.model_to_cost == {"model-3": 3e-06} + + +def test_delete_of_an_unknown_deployment_returns_none_and_keeps_the_list() -> None: + router: Final = _concurrent_delete_router() + + assert router.delete_deployment("deployment-9") is None + assert router.get_model_ids() == [f"deployment-{index}" for index in range(4)] + + +def test_get_model_ids_by_name_skips_team_deployments_only_when_asked() -> None: + shared: Final = {"model_name": "shared", "litellm_params": {"model": "openai/gpt-shared", "api_key": "k"}} + router: Final = Router( + model_list=[ + {**shared, "model_info": {"id": "global"}}, + {**shared, "model_info": {"id": "team", "team_id": "team-1", "team_public_model_name": "public"}}, + ] + ) + + assert router.get_model_ids("shared", exclude_team_models=True) == ["global"] + assert router.get_model_ids("shared") == ["global", "team"] + + _IMAGE_PART = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}} _IMAGE_CHAT = [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, _IMAGE_PART]}] _IMAGE_RESPONSES_INPUT = [ @@ -30149,3 +30509,84 @@ def test_request_carries_image_sees_chat_parts_and_responses_input_items(): assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("isolate_litellm_state") +@pytest.mark.parametrize("surface", ["chat", "messages"]) +@pytest.mark.parametrize("credential_kind", ["oauth", "api_key"]) +async def test_anthropic_caller_rate_limit_does_not_block_other_credentials( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, surface: str, credential_kind: str +) -> None: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.types.router import AllowedFailsPolicy + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr("time.time", lambda: 1767225600.0) + + def upstream(request: httpx.Request) -> httpx.Response: + credential: Final = request.headers.get("authorization", request.headers.get("x-api-key", "")) + if credential.endswith("limited") and route.call_count < 2: + return httpx.Response( + 429, + headers={"retry-after": "168307"}, + json={"type": "error", "error": {"type": "rate_limit_error", "message": "usage limit"}}, + ) + return httpx.Response(200, json={ + "id": "msg-healthy", "type": "message", "role": "assistant", "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "healthy account"}], "stop_reason": "end_turn", + "usage": {"input_tokens": 2, "output_tokens": 2}, + }) + + route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock(side_effect=upstream) + router: Final = Router( + model_list=[{ + "model_name": "claude", "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", "api_key": "fake-gateway-key", + }, "model_info": {"id": "shared-anthropic"}, + }], + num_retries=0, + allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1), + ) + client: Final = AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()) + + async def call(account: str | None) -> object: + credentials: Final = ( + {"provider_specific_header": { + "custom_llm_provider": "anthropic", "extra_headers": { + "Authorization": f"Bearer sk-ant-oat-fake-{account}", + }, + }} if credential_kind == "oauth" else {"api_key": f"fake-{account}"} + ) if account is not None else {} + if surface == "messages": + max_tokens: Final = litellm.get_max_tokens("anthropic/claude-sonnet-5-5") + assert max_tokens is not None + return await router.aanthropic_messages( + model="claude", messages=[{"role": "user", "content": "hello"}], + client=client, max_tokens=max_tokens, **credentials, + ) + return await router.acompletion( + model="claude", messages=[{"role": "user", "content": "hello"}], + client=client, **credentials, + ) + + try: + for _ in range(2): + with pytest.raises(litellm.RateLimitError): + await call("limited") + await GLOBAL_LOGGING_WORKER.flush() + limited_calls: Final = len(route.calls) + assert limited_calls == 2 + await call("healthy") + assert len(route.calls) == limited_calls + 1 + assert route.calls[-1].request.headers.get( + "authorization", route.calls[-1].request.headers.get("x-api-key", "") + ) == ("Bearer sk-ant-oat-fake-healthy" if credential_kind == "oauth" else "fake-healthy") + await call(None) + assert len(route.calls) == limited_calls + 2 + assert route.calls[-1].request.headers["x-api-key"] == "fake-gateway-key" + await call("limited") + assert len(route.calls) == limited_calls + 3 + finally: + await GLOBAL_LOGGING_WORKER.flush() + await client.close() diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index f001008ee2c..20b0055f729 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -3081,6 +3081,36 @@ def test_reload_keeps_custom_pricing_configured_on_litellm_params_for_a_db_model _invalidate_model_cost_lowercase_map() +def test_reload_keeps_custom_pricing_of_a_db_model_on_a_router_built_without_a_model_list( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ + A proxy whose config.yaml has no model_list builds its router without one and + takes every deployment from the DB afterwards. A price data reload must restore + that deployment's custom pricing exactly as it does for a config router. + """ + fetched_catalog: Final = copy.deepcopy(litellm.model_cost) + router: Final = Router() + router.add_deployment( + deployment=Deployment( + model_name="db-only-priced", + litellm_params=LiteLLM_Params( + model="openai/gpt-4o", + api_key="sk-fake", + input_cost_per_token=0.000123, + output_cost_per_token=0.000456, + ), + model_info=ModelInfo(id="db-only-priced-id"), + ) + ) + assert litellm.model_cost["db-only-priced-id"]["input_cost_per_token"] == 0.000123 + + _simulate_price_data_reload_with_provider_sets(monkeypatch, fetched_catalog) + + assert litellm.model_cost["db-only-priced-id"]["input_cost_per_token"] == 0.000123 + assert litellm.model_cost["db-only-priced-id"]["output_cost_per_token"] == 0.000456 + + def test_replay_live_router_model_cost_rebuilds_every_live_router(): """ A process can hold more than one Router, so the rebuild has to fan out across diff --git a/tests/unit/test_sync_together_ai_models.py b/tests/unit/test_sync_together_ai_models.py deleted file mode 100644 index b8a85bcfbdc..00000000000 --- a/tests/unit/test_sync_together_ai_models.py +++ /dev/null @@ -1,378 +0,0 @@ -import importlib.util -import json -from pathlib import Path -from types import MappingProxyType - -import pytest - -ROOT = Path(__file__).resolve().parents[2] -SCRIPT = ROOT / "scripts" / "sync_together_ai_models.py" -FIXTURES = Path(__file__).resolve().parent / "fixtures" / "together_ai_sync" - -_spec = importlib.util.spec_from_file_location("sync_together_ai_models", SCRIPT) -assert _spec is not None and _spec.loader is not None -sync = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(sync) - -RECORDED_CATALOG = sync.load_catalog(FIXTURES.joinpath("models_serverless.json").read_bytes()) -RECORDED_DOC = sync.parse_deprecations(FIXTURES.joinpath("deprecations.md").read_text()) - - -def _doc(removal_dates: dict[str, str], redirects: dict[str, str] | None = None) -> object: - return sync.DeprecationDoc( - removal_dates=MappingProxyType(removal_dates), - redirects=MappingProxyType(redirects or {}), - ) - - -def _chat_model(model_id: str, ctx: int = 4096, price: float = 1.0, cached: float | None = None) -> object: - return sync.CatalogModel( - id=model_id, - type="chat", - context_length=ctx, - pricing=sync.CatalogPricing(input=price, output=price, cached_input=cached), - ) - - -@pytest.mark.parametrize( - ("per_million", "expected"), - [ - (3, 3e-06), - (15, 1.5e-05), - (1.4, 1.4e-06), - (0.25999999999999995, 2.6e-07), - (0.060000000000000005, 6e-08), - (1.0399999999999998, 1.04e-06), - (0, 0.0), - ], -) -def test_per_token_normalizes_float_artifacts(per_million: float, expected: float) -> None: - assert sync.per_token(per_million) == expected - - -def test_parse_deprecations_recorded_fixture() -> None: - assert dict(RECORDED_DOC.redirects) == { - "mistralai/Mistral-7B-Instruct-v0.3": "mistralai/Ministral-3-14B-Instruct-2512", - "Kimi-K2": "Kimi-K2-0905", - "DeepSeek-V3": "DeepSeek-V3.1", - "DeepSeek-V3-0324": "DeepSeek-V3.1", - "DeepSeek-R1": "DeepSeek-R1-0528", - } - assert len(RECORDED_DOC.removal_dates) == 208 - assert RECORDED_DOC.removal_dates["google/gemma-3n-E4B-it"] == "2026-08-04" - - -def test_parse_deprecations_duplicate_rows_keep_most_recent_date() -> None: - assert RECORDED_DOC.removal_dates["Qwen/Qwen3-235B-A22B-Thinking-2507"] == "2026-04-16" - - -@pytest.mark.parametrize( - "markdown", - [ - "# Deprecations\n\nNothing here anymore.\n", - "\n## Active model redirects\n\n| A | B |\n| --- | --- |\n| `x` | `y` |\n\n## Something else\n", - "\n## Deprecation history\n\n### Inference\n\n| Date | Model | R |\n| --- | --- | --- |\n| 2026-01-01 | `m` | No |\n", - ], -) -def test_parse_deprecations_raises_when_a_table_parses_empty(markdown: str) -> None: - with pytest.raises(sync.SyncError): - sync.parse_deprecations(markdown) - - -def test_load_catalog_raises_on_shape_change() -> None: - with pytest.raises(sync.SyncError): - sync.load_catalog(b'[{"id": "x", "type": "chat"}]') - - -def test_load_catalog_raises_when_no_token_models_remain() -> None: - only_video = json.dumps([{"id": "v", "type": "video", "pricing": {"input": 0, "output": 0}}]).encode() - with pytest.raises(sync.SyncError): - sync.load_catalog(only_video) - - -def test_recorded_catalog_counts() -> None: - assert len(RECORDED_CATALOG) == 102 - assert sum(1 for model in RECORDED_CATALOG if model.type in sync.TYPE_TO_MODE) == 26 - assert sum(1 for model in RECORDED_CATALOG if model.pricing.cached_input) == 13 - - -def test_added_chat_model_matches_reviewed_registry_shape() -> None: - outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) - assert len(outcome.added) == 26 - assert not outcome.deprecated - assert outcome.cost_map["together_ai/moonshotai/Kimi-K3"] == { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 1048576, - "max_tokens": 1048576, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "source": "https://docs.together.ai/docs/serverless-models", - "supports_function_calling": True, - "supports_parallel_function_calling": True, - "supports_prompt_caching": True, - "supports_reasoning": True, - "supports_response_schema": True, - "supports_tool_choice": True, - "supports_vision": True, - } - - -def test_added_embedding_model_has_no_output_token_cap() -> None: - outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) - assert outcome.cost_map["together_ai/intfloat/multilingual-e5-large-instruct"] == { - "input_cost_per_token": 2e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 514, - "max_tokens": 514, - "mode": "embedding", - "output_cost_per_token": 2e-08, - "output_vector_size": 1024, - "source": "https://docs.together.ai/docs/serverless-models", - } - - -def test_moderation_type_maps_to_chat_mode() -> None: - outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) - guard = outcome.cost_map["together_ai/meta-llama/Llama-Guard-4-12B"] - assert guard["mode"] == "chat" - assert "max_output_tokens" not in guard - - -def test_output_ceiling_comes_from_the_rule_never_from_context_length() -> None: - glm = next(model for model in RECORDED_CATALOG if model.id == "zai-org/GLM-5.2") - fresh = sync.compute_sync({}, [_chat_model("acme/unreviewed", ctx=1048576), glm], _doc({"x": "2026-01-01"})) - unreviewed = fresh.cost_map["together_ai/acme/unreviewed"] - assert "max_output_tokens" not in unreviewed - assert (unreviewed["max_input_tokens"], unreviewed["max_tokens"]) == (1048576, 1048576) - reviewed = fresh.cost_map["together_ai/zai-org/GLM-5.2"] - assert (reviewed["max_input_tokens"], reviewed["max_output_tokens"], reviewed["max_tokens"]) == ( - 1048575, - 128000, - 128000, - ) - inflated = { - "together_ai/zai-org/GLM-5.2": { - "input_cost_per_token": 1.4e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 1048575, - "max_output_tokens": 1048575, - "max_tokens": 1048575, - "mode": "chat", - "output_cost_per_token": 4.4e-06, - } - } - corrected = sync.compute_sync(inflated, [glm], _doc({"x": "2026-01-01"})) - assert corrected.cost_map["together_ai/zai-org/GLM-5.2"]["max_output_tokens"] == 128000 - assert any("max_output_tokens: 1048575 -> 128000" in line for line in corrected.updated) - - -def test_docs_removed_but_live_model_stays_live_with_warning() -> None: - outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) - gemma = outcome.cost_map["together_ai/google/gemma-3n-E4B-it"] - assert "deprecation_date" not in gemma - assert any("gemma-3n-E4B-it" in warning and "2026-08-04" in warning for warning in outcome.warnings) - - -def test_price_change_updates_api_fields_and_keeps_curated_ones() -> None: - registry = { - "together_ai/acme/chat-1": { - "input_cost_per_token": 9e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 4096, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 9e-07, - "supports_audio_input": True, - } - } - outcome = sync.compute_sync(registry, [_chat_model("acme/chat-1", ctx=8192, price=2.0)], _doc({"x": "2026-01-01"})) - entry = outcome.cost_map["together_ai/acme/chat-1"] - assert entry["input_cost_per_token"] == 2e-06 - assert entry["max_input_tokens"] == 8192 - assert entry["max_output_tokens"] == 2048 - assert entry["supports_audio_input"] is True - assert len(outcome.updated) == 1 - assert "input_cost_per_token" in outcome.updated[0] - - -def test_cached_input_appearing_and_disappearing() -> None: - registry = { - "together_ai/acme/chat-1": { - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1e-06, - "supports_prompt_caching": True, - }, - "together_ai/acme/chat-2": { - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1e-06, - }, - } - catalog = [_chat_model("acme/chat-1"), _chat_model("acme/chat-2", cached=0.25999999999999995)] - outcome = sync.compute_sync(registry, catalog, _doc({"x": "2026-01-01"})) - assert "cache_read_input_token_cost" not in outcome.cost_map["together_ai/acme/chat-1"] - assert "supports_prompt_caching" not in outcome.cost_map["together_ai/acme/chat-1"] - assert outcome.cost_map["together_ai/acme/chat-2"]["cache_read_input_token_cost"] == 2.6e-07 - assert outcome.cost_map["together_ai/acme/chat-2"]["supports_prompt_caching"] is True - - -def test_capability_rule_backfills_existing_entry() -> None: - registry = { - "together_ai/moonshotai/Kimi-K3": { - "input_cost_per_token": 3e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 1048576, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - } - } - kimi = next(model for model in RECORDED_CATALOG if model.id == "moonshotai/Kimi-K3") - outcome = sync.compute_sync(registry, [kimi], _doc({"x": "2026-01-01"})) - assert outcome.cost_map["together_ai/moonshotai/Kimi-K3"]["supports_reasoning"] is True - assert any("supports_reasoning" in line for line in outcome.updated) - - -def test_disappeared_model_gets_docs_date_and_is_never_deleted() -> None: - registry = { - "together_ai/acme/gone": { - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1e-06, - } - } - outcome = sync.compute_sync(registry, [_chat_model("acme/alive")], _doc({"acme/gone": "2026-07-01"})) - assert outcome.cost_map["together_ai/acme/gone"]["deprecation_date"] == "2026-07-01" - assert outcome.deprecated == ("together_ai/acme/gone: deprecation_date",) - - -def test_disappeared_model_without_docs_date_warns_instead() -> None: - registry = { - "together_ai/acme/gone": { - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1e-06, - } - } - outcome = sync.compute_sync(registry, [_chat_model("acme/alive")], _doc({"other": "2026-07-01"})) - assert "deprecation_date" not in outcome.cost_map["together_ai/acme/gone"] - assert not outcome.deprecated - assert any("acme/gone" in warning and "human" in warning for warning in outcome.warnings) - - -def test_curated_deprecation_date_is_never_overwritten() -> None: - registry = { - "together_ai/acme/gone": { - "deprecation_date": "2026-06-15", - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1e-06, - } - } - outcome = sync.compute_sync(registry, [_chat_model("acme/alive")], _doc({"acme/gone": "2026-07-01"})) - assert outcome.cost_map["together_ai/acme/gone"]["deprecation_date"] == "2026-06-15" - assert any("2026-06-15" in warning and "2026-07-01" in warning for warning in outcome.warnings) - - -def test_redirect_chain_resolves_to_final_live_model() -> None: - doc = _doc({"acme/a": "2026-01-01"}, redirects={"acme/a": "acme/b", "acme/b": "acme/c"}) - live = frozenset({"acme/c"}) - assert sync.resolve_successor("acme/a", doc, live) == "acme/c" - - -def test_redirect_dead_end_yields_no_successor() -> None: - doc = _doc({"acme/a": "2026-01-01"}, redirects={"acme/a": "acme/b"}) - assert sync.resolve_successor("acme/a", doc, frozenset({"acme/other"})) is None - - -def test_redirect_short_names_resolve_by_unique_suffix() -> None: - doc = _doc({"moonshotai/Kimi-K2": "2026-01-01"}, redirects={"Kimi-K2": "Kimi-K2-0905"}) - live = frozenset({"moonshotai/Kimi-K2-0905"}) - assert sync.resolve_successor("moonshotai/Kimi-K2", doc, live) == "moonshotai/Kimi-K2-0905" - - -def test_successor_written_only_when_not_curated() -> None: - registry = { - "together_ai/acme/a": { - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1e-06, - }, - "together_ai/acme/b": { - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "metadata": {"successor": "together_ai/acme/curated"}, - "mode": "chat", - "output_cost_per_token": 1e-06, - }, - } - doc = _doc({"acme/a": "2026-01-01", "acme/b": "2026-01-01"}, redirects={"acme/a": "acme/c", "acme/b": "acme/c"}) - outcome = sync.compute_sync(registry, [_chat_model("acme/c")], doc) - assert outcome.cost_map["together_ai/acme/a"]["metadata"] == {"successor": "together_ai/acme/c"} - assert outcome.cost_map["together_ai/acme/b"]["metadata"] == {"successor": "together_ai/acme/curated"} - assert any("acme/curated" in warning for warning in outcome.warnings) - - -def test_reappearance_clears_deprecation_date() -> None: - registry = { - "together_ai/acme/back": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1e-06, - } - } - outcome = sync.compute_sync(registry, [_chat_model("acme/back")], _doc({"x": "2026-01-01"})) - assert "deprecation_date" not in outcome.cost_map["together_ai/acme/back"] - assert outcome.reappeared == ("together_ai/acme/back",) - - -def test_new_chat_model_without_rule_is_flagged() -> None: - outcome = sync.compute_sync({}, [_chat_model("acme/unreviewed")], _doc({"x": "2026-01-01"})) - assert any("acme/unreviewed" in warning and "capability rule" in warning for warning in outcome.warnings) - - -def test_new_keys_land_at_the_end_of_the_provider_block() -> None: - registry = { - "aaa": {"mode": "chat"}, - "together_ai/acme/old": { - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1e-06, - }, - "zzz": {"mode": "chat"}, - } - outcome = sync.compute_sync(registry, [_chat_model("acme/old"), _chat_model("acme/new")], _doc({"x": "2026-01-01"})) - assert list(outcome.cost_map) == ["aaa", "together_ai/acme/old", "together_ai/acme/new", "zzz"] - - -def test_sync_is_idempotent_over_the_repo_cost_map() -> None: - cost_map = json.loads((ROOT / "model_prices_and_context_window.json").read_text()) - first = sync.compute_sync(cost_map, RECORDED_CATALOG, RECORDED_DOC) - second = sync.compute_sync(first.cost_map, RECORDED_CATALOG, RECORDED_DOC) - assert not second.has_changes - assert second.cost_map == first.cost_map - - -def test_pr_body_lists_every_section_and_the_skipped_types() -> None: - outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) - body = sync.render_pr_body(outcome) - assert "### Added (26)" in body - assert "### Warnings needing a human call" in body - assert "image (29)" in body - assert "video (38)" in body diff --git a/tests/unit/test_together_ai_model_metadata.py b/tests/unit/test_together_ai_model_metadata.py index dccba3d66f9..3616c6216ca 100644 --- a/tests/unit/test_together_ai_model_metadata.py +++ b/tests/unit/test_together_ai_model_metadata.py @@ -75,18 +75,6 @@ def cost_map() -> CostMap: return COST_MAP_ADAPTER.validate_python(json.load(f)) -def test_together_chat_entries_never_carry_context_length_as_output_ceiling(cost_map: CostMap): - inflated = sorted( - model - for model, info in cost_map.items() - if info.get("litellm_provider") == "together_ai" - and info.get("mode") == "chat" - and "max_output_tokens" in info - and info["max_output_tokens"] == info.get("max_input_tokens") - ) - assert inflated == [] - - def _successor(info: dict[str, object]) -> str | None: metadata = info.get("metadata") if not isinstance(metadata, dict): @@ -95,8 +83,6 @@ def _successor(info: dict[str, object]) -> str | None: return successor if isinstance(successor, str) else None - - def test_together_backup_cost_map_in_sync(cost_map: CostMap): with open(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") as f: backup = COST_MAP_ADAPTER.validate_python(json.load(f)) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 55b7bc78a11..38287b25169 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -576,7 +576,7 @@ def test_all_model_configs(): from litellm.llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import ( VertexAIAi21Config, ) - from litellm.llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import ( + from litellm.llms.vertex_ai.vertex_ai_partner_models.llama.transformation import ( VertexAILlama3Config, ) @@ -1086,6 +1086,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "chat", "completion", "container", + "decisions", "image_edit", "embedding", "evaluation", @@ -2259,6 +2260,45 @@ class TestProxyFunctionCalling: @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") +def test_get_model_info_derives_max_tokens_from_the_output_limit() -> None: + model: Final = "derived-max-tokens-chat-model" + litellm.register_model( + { + model: { + "litellm_provider": "openai", + "mode": "chat", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + }, + persist_across_reloads=False, + ) + info: Final = litellm.get_model_info(model, custom_llm_provider="openai") + assert info["max_tokens"] == 128000 + assert litellm.get_max_tokens(model) == 128000 + + +def test_get_model_info_derives_max_tokens_from_the_input_limit_when_the_row_has_no_output_limit() -> None: + model: Final = "derived-max-tokens-embedding-model" + litellm.register_model( + { + model: { + "litellm_provider": "openai", + "mode": "embedding", + "max_input_tokens": 8191, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + }, + persist_across_reloads=False, + ) + info: Final = litellm.get_model_info(model, custom_llm_provider="openai") + assert info["max_tokens"] == 8191 + assert info["max_output_tokens"] is None + + def test_register_model_price_is_used_for_completion_cost() -> None: model: Final = "migration-custom-pricing-model" price: Final = { @@ -2570,7 +2610,7 @@ def test_priced_router_deployment_id_stays_out_of_wildcard_model_listing(): def test_anthropic_claude_4_invoke_chat_provider_config(): """Test that the Anthropic Claude 4 Invoke chat provider config is correct.""" - from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) @@ -5116,7 +5156,7 @@ def test_request_path_cost_map_walks_survive_a_registration_mid_walk( from litellm.llms.fal_ai.cost_calculator import _keyed_rows from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig from litellm.llms.ollama.common_utils import OllamaModelInfo - from litellm.llms.openai.chat.gpt_5_transformation import _catalogue_declares_default_effort + from litellm.llms.openai.chat.gpt_reasoning_transformation import _catalogue_declares_default_effort from litellm.utils import _handle_new_key_with_scan, _invalidate_model_cost_lowercase_map walks: Final[Mapping[str, Callable[[], object]]] = { @@ -8628,7 +8668,7 @@ def test_basic_trimming_no_max_tokens_specified(): trimmed_messages = trim_messages(messages, model="gpt-4") print("trimmed messages for gpt-4") print(trimmed_messages) - assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"]["max_tokens"] + assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"]["max_input_tokens"] def test_multiple_messages_trimming(): diff --git a/tests/unit/test_uuid_helper.py b/tests/unit/test_uuid_helper.py index 589d3f96a17..f636942ba88 100644 --- a/tests/unit/test_uuid_helper.py +++ b/tests/unit/test_uuid_helper.py @@ -1,14 +1,20 @@ -import importlib +from typing import Final +from uuid import NAMESPACE_DNS, UUID, uuid5 as stdlib_uuid5 + +from litellm._uuid import uuid, uuid4 -def test_uses_fastuuid_and_uuid4_works(): - mod = importlib.import_module("litellm._uuid") - fastuuid_mod = importlib.import_module("fastuuid") - assert hasattr(mod, "uuid4") - assert hasattr(mod, "uuid") - assert mod.uuid is fastuuid_mod +def test_uuid4_returns_canonical_random_uuid() -> None: + for value in (uuid4(), uuid.uuid4()): + encoded: Final = str(value) + parsed: Final = UUID(encoded) + assert parsed.version == 4 + assert encoded == str(parsed) + assert value.bytes == parsed.bytes - # Ensure uuid4 returns something that looks like a UUID string - val = str(mod.uuid4()) - assert isinstance(val, str) - assert len(val) >= 8 + +def test_uuid5_accepts_byte_names_for_stable_ids() -> None: + namespace: Final = uuid.UUID(str(NAMESPACE_DNS)) + name: Final = "guardrail-\u00e9" + generated: Final = uuid.uuid5(namespace, name.encode("utf-8")) + assert str(generated) == str(stdlib_uuid5(NAMESPACE_DNS, name)) diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index b0086662f1c..740342f54d9 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -94,6 +94,7 @@ CONNECTION_NAMES: Final = ( "github_copilot_host", "configurable_clientside_auth_params", "use_xai_oauth", + "chatgpt_auth_file", "fireworks_forward_user_id", "aws_batch_role_arn", "aws_bedrock_control_plane_endpoint", diff --git a/tests/unit/types/test_router.py b/tests/unit/types/test_router.py index 2892a829eab..7855eba0042 100644 --- a/tests/unit/types/test_router.py +++ b/tests/unit/types/test_router.py @@ -16,8 +16,8 @@ from litellm.types.router import ( holds_secret_pointer, reject_server_owned_wif_params, server_owned_wif_fields_named, - server_owned_wif_fields_written, server_owned_wif_fields_present, + server_owned_wif_fields_written, ) from litellm.types.utils import ( CustomPricingLiteLLMParams, @@ -367,6 +367,7 @@ def test_server_owned_registry_includes_anthropic_openai_and_oauth_token_exchang + openai_wif_litellm_params + oauth_token_exchange_litellm_params + github_copilot_oauth_litellm_params + + ("chatgpt_auth_file",) ) assert set(openai_wif_litellm_params) == { "openai_identity_provider_id", diff --git a/ui/Dockerfile b/ui/Dockerfile index 4b6b0c1581a..d78a155b874 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -3,7 +3,7 @@ # UI container — Next.js static export served by nginx. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 -ARG NGINX_VERSION=1.31.5-alpine3.24@sha256:34f40471dea485273c5e2a04dd5e97a682332ceb4a9adecd67de450dcb2fb390 +ARG NGINX_VERSION=1.31.8-alpine3.24@sha256:1ef7ca94d7879df82421428ce13bd1eaa412404953859cf7a5b6512c25e67a60 # ---------- builder ---------- FROM ${UI_BUILD_IMAGE} AS builder @@ -27,6 +27,8 @@ RUN npm run build # ---------- runtime ---------- FROM nginx:${NGINX_VERSION} AS runtime +RUN apk upgrade --no-cache brotli-libs + # Drop the upstream default :80 server; we own the config. RUN rm -f /etc/nginx/conf.d/default.conf diff --git a/ui/litellm-dashboard/AGENTS.md b/ui/litellm-dashboard/AGENTS.md index bbf2e49aad7..b6fe5219022 100644 --- a/ui/litellm-dashboard/AGENTS.md +++ b/ui/litellm-dashboard/AGENTS.md @@ -30,7 +30,7 @@ Type tests are `*.test-d.ts` files run by the `types` vitest project (`npm run t -## This is NOT the Next.js you know +# This is NOT the Next.js you know This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices. diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 38d4c435e9a..c8231b90bfd 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1730,9 +1730,6 @@ "src/components/model_group_alias_settings.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "react-hooks/set-state-in-effect": { - "count": 1 } }, "src/components/model_info_view.tsx": { @@ -1740,7 +1737,7 @@ "count": 1 }, "prefer-const": { - "count": 5 + "count": 3 }, "react-hooks/set-state-in-effect": { "count": 1 @@ -2175,11 +2172,6 @@ "count": 1 } }, - "src/utils/dataUtils.test.ts": { - "max-nested-callbacks": { - "count": 1 - } - }, "tests/CreateKeyPage.expiredToken.test.tsx": { "@typescript-eslint/no-require-imports": { "count": 3 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.test.ts index e9df939f220..76db710f5ba 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.test.ts @@ -125,10 +125,42 @@ describe("useModelsInfo", () => { undefined, false, undefined, + {}, ); expect(modelInfoCall).toHaveBeenCalledTimes(1); }); + it("keeps saved model parameters separate from hydrated parameters in the query cache", async () => { + vi.mocked(modelInfoCall).mockImplementation(async (...args) => ({ + ...mockPaginatedModelInfoResponse, + data: [{ litellm_params: args[15]?.useStoredParams ? {} : { litellm_image_enabled: false } }], + })); + const { result } = renderHook( + () => ({ + hydrated: useModelsInfo(1, 50, undefined, "model-1"), + stored: useModelsInfo( + 1, + 50, + undefined, + "model-1", + undefined, + undefined, + undefined, + false, + undefined, + undefined, + false, + undefined, + { useStoredParams: true }, + ), + }), + { wrapper }, + ); + await waitFor(() => expect(result.current.stored.isSuccess && result.current.hydrated.isSuccess).toBe(true)); + expect(result.current.stored.data?.data[0].litellm_params).toEqual({}); + expect(result.current.hydrated.data?.data[0].litellm_params).toEqual({ litellm_image_enabled: false }); + }); + it("should use custom page and size parameters", async () => { (modelInfoCall as any).mockResolvedValue(mockPaginatedModelInfoResponse); @@ -156,6 +188,7 @@ describe("useModelsInfo", () => { undefined, false, undefined, + {}, ); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index 24f76065f34..f9304bab0d2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -45,6 +45,7 @@ export const useModelsInfo = ( accessGroup?: string, wildcardOnly: boolean = false, blocked?: boolean, + options: { useStoredParams?: boolean } = {}, ) => { const { accessToken, userId, userRole } = useAuthorized(); return useQuery({ @@ -57,6 +58,7 @@ export const useModelsInfo = ( ...(search && { search }), ...(modelName && { modelName }), ...(modelId && { modelId }), + useStoredParams: Boolean(options.useStoredParams), ...(teamId && { teamId }), ...(sortBy && { sortBy }), ...(sortOrder && { sortOrder }), @@ -86,6 +88,7 @@ export const useModelsInfo = ( accessGroup, wildcardOnly, blocked, + options, ), enabled: Boolean(accessToken && userId && userRole), }); @@ -121,6 +124,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { /** False for config.yaml-defined deployments, which the update and delete routes refuse. */ db_model?: boolean | null; base_model?: string | null; + blocked?: boolean | null; mode?: string | null; created_at?: string | null; updated_at?: string | null; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx index 6f7e04f92c6..2d1c24685f0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx @@ -106,10 +106,7 @@ export function AutoRoutersPanel({ > Add Auto Router - - Choose a classifier to route each request to a model. Called like any other model, so clients keep using a - single model name. - + Choose models for your auto router. ({ vi.mock("@/components/networking", () => ({ getCallbacksCall: vi.fn(), + getProxyBaseUrl: () => "https://proxy.test", getRouterSettingsCall: vi.fn(), setCallbacksCall: vi.fn(), })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelGroupAliasPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelGroupAliasPanel.tsx index 025923e08e9..e6c720f38cf 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelGroupAliasPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/ModelGroupAliasPanel.tsx @@ -2,7 +2,7 @@ import { useEffect, useState } from "react"; import ModelGroupAliasSettings from "@/components/model_group_alias_settings"; -import { getCallbacksCall, getRouterSettingsCall } from "@/components/networking"; +import { getCallbacksCall, getProxyBaseUrl, getRouterSettingsCall } from "@/components/networking"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function ModelGroupAliasPanel() { @@ -22,6 +22,7 @@ export default function ModelGroupAliasPanel() { : null; const managedByConfig = currentAliasOwnership?.managedByConfig ?? false; const ownershipLoading = Boolean(accessToken && !currentAliasOwnership); + const preferenceScope = userID ? JSON.stringify([getProxyBaseUrl(), userID]) : null; useEffect(() => { if (!accessToken) { @@ -62,6 +63,8 @@ export default function ModelGroupAliasPanel() { return ( { vi.unstubAllGlobals(); }); - it("lists the key's decision models from the proxy that answers the request, sorted by name", async () => { + it("lists the key's models with decision modes or endpoints from the request proxy, sorted by name", async () => { mockFetch.mockResolvedValueOnce( modelGroupInfo([ { model_group: "pplx-decider", mode: "evaluation" }, { model_group: "gpt-5.5", mode: "chat" }, { model_group: "jev-latest", mode: "evaluation" }, { model_group: "unknown-mode", mode: null }, + { model_group: "native-decision", mode: "decisions" }, + { + model_group: "chat-decisions", + mode: "chat", + supported_endpoints: ["/v1/chat/completions", "/v1/decisions"], + }, + { model_group: "chat-systemone", mode: "chat", supported_endpoints: ["/v1/systemone"] }, + { model_group: "custom-decision", mode: "custom", supported_endpoints: ["/v1/decisions"] }, + { model_group: "endpoint-only", supported_endpoints: ["/v1/decisions"] }, + { model_group: "chat-only", mode: "chat", supported_endpoints: ["/v1/chat/completions"] }, + { model_group: "no-endpoints", mode: "chat", supported_endpoints: [] }, + { model_group: "null-endpoints", mode: "chat", supported_endpoints: null }, + { model_group: "missing-mode" }, + { model_group: "custom-mode", mode: "custom" }, ]), ); await expect(fetchDecisionModels("virtual-key", "https://custom.example.com/")).resolves.toEqual([ + "chat-decisions", + "chat-systemone", + "custom-decision", + "endpoint-only", "jev-latest", + "native-decision", "pplx-decider", ]); expect(mockFetch).toHaveBeenCalledWith( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts index a4030029903..035a05729ba 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { getGatewayBaseUrl, getGlobalLitellmHeaderName, getProxyBaseUrl } from "@/components/networking"; import { withRequiredHeaders } from "@/components/llm_calls/request_headers"; import { createApiClient } from "@/lib/http/client"; -import { isDecisionMode } from "@/lib/decisionModels"; +import { isDecisionEntry } from "@/lib/decisionModels"; import { systemOneResponseSchema, type PlaygroundRequest, @@ -26,7 +26,13 @@ export type PlaygroundDecisionRequest = | { endpoint: "/v1/decisions"; payload: OpenAIDecisionsRequest }; const modelGroupInfoSchema = z.object({ - data: z.array(z.object({ model_group: z.string(), mode: z.string().nullish() })), + data: z.array( + z.object({ + model_group: z.string(), + mode: z.string().nullish(), + supported_endpoints: z.array(z.string()).nullish(), + }), + ), }); function proxyClient(customBaseUrl?: string, getDefaultBaseUrl: () => string = getProxyBaseUrl) { @@ -56,7 +62,7 @@ export async function fetchDecisionModels( }); return modelGroupInfoSchema .parse(body) - .data.filter((group) => isDecisionMode(group.mode)) + .data.filter(isDecisionEntry) .map((group) => group.model_group) .sort((a, b) => a.localeCompare(b)); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx index 35434830d98..d7dfb8151fc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx @@ -80,6 +80,42 @@ describe("GeneralSettings General tab", () => { vi.mocked(deleteConfigFieldSetting).mockClear(); }); + it.each([ + ["maximum_spend_logs_retention_period", null, "7d"], + ["maximum_health_check_retention_period", "30d", "14d"], + ["maximum_daily_tag_spend_retention_period", undefined, "90d"], + ["maximum_spend_logs_cleanup_run_budget", "5m", "10m"], + ["maximum_spend_logs_cleanup_batch_timeout", "", "45s"], + ["mcp_client_id_header", "x-client-id", "X-MCP-Client.ID"], + ])("displays, updates and resets the String setting %s", async (fieldName, initialValue, updatedValue) => { + vi.mocked(getGeneralSettingsCall).mockResolvedValue([ + ...SETTINGS_FIXTURE, + { + field_name: fieldName, + field_type: "String", + field_value: initialValue, + field_description: "string setting", + stored_in_db: initialValue != null, + field_default_value: null, + }, + ]); + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("tab", { name: "General" })); + const input = await screen.findByRole("textbox", { name: fieldName }); + expect(input).toHaveValue(initialValue ?? ""); + fireEvent.change(input, { target: { value: updatedValue } }); + const row = screen.getByRole("row", { name: new RegExp(fieldName) }); + await user.click(within(row).getByRole("button", { name: "Update" })); + expect(vi.mocked(updateConfigFieldSetting).mock.calls).toEqual([["token", fieldName, updatedValue]]); + expect(within(row).getByText("In DB")).toBeInTheDocument(); + const resetIcon = row.querySelectorAll("td")[3].querySelector("svg"); + expect(resetIcon).not.toBeNull(); + await user.click(resetIcon as unknown as Element); + expect(vi.mocked(deleteConfigFieldSetting).mock.calls).toEqual([["token", fieldName]]); + expect(input).toHaveValue(""); + }); + it("updates max_ui_session_budget with its own value, not the value at its filtered index", async () => { const user = userEvent.setup(); renderWithProviders(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index 5d28adb7ef3..67712d78a13 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -68,6 +68,15 @@ const SettingValueEditor: React.FC<{ /> ); } + if (setting.field_type === "String") { + return ( + onChange(setting.field_name, event.target.value)} + /> + ); + } if (setting.field_type === "Integer") { return ( { }); it("should handle search errors and show notification", async () => { - const error = new Error("Search failed"); + const error = new Error( + JSON.stringify({ + error: { + message: "User not allowed to access search tool: exa-search-2", + type: "user_search_tool_access_denied", + code: "403", + }, + }), + ); vi.mocked(networking.searchToolQueryCall).mockRejectedValue(error); const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); const user = userEvent.setup(); @@ -281,11 +289,29 @@ describe("SearchToolTester", () => { const searchButton = screen.getByRole("button", { name: /search/i }); await user.click(searchButton); await waitFor(() => { - expect(toast.fromError).toHaveBeenCalledWith("Failed to query search tool"); + expect(toast.fromError).toHaveBeenCalledWith(error); }); consoleSpy.mockRestore(); }); + it("should not show empty results or add history when search fails", async () => { + vi.mocked(networking.searchToolQueryCall).mockRejectedValue(new Error("Search failed")); + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const user = userEvent.setup(); + render(); + const input = screen.getByPlaceholderText("Enter your search query..."); + fireEvent.change(input, { target: { value: "test query" } }); + await user.click(screen.getByRole("button", { name: /search/i })); + + await waitFor(() => { + expect(toast.fromError).toHaveBeenCalled(); + }); + expect(screen.queryByText("No results found")).not.toBeInTheDocument(); + expect(screen.queryByText("test query")).not.toBeInTheDocument(); + expect(screen.getByText("Test your search tool")).toBeInTheDocument(); + consoleSpy.mockRestore(); + }); + it("should maintain search history", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolTester.tsx b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolTester.tsx index 7dcef3fc6ba..646120443c7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolTester.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolTester.tsx @@ -60,7 +60,7 @@ export const SearchToolTester: React.FC = ({ searchToolNa setSearchHistory((prev) => [historyEntry, ...prev]); } catch (error) { console.error("Error querying search tool:", error); - toast.fromError("Failed to query search tool"); + toast.fromError(error); } finally { setIsLoading(false); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx index 2056142b50a..476c043a258 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx @@ -47,6 +47,7 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/u import ScopedSavingsTab from "@/components/shared/ScopedSavingsTab"; import { AutoRouterUsageView } from "@/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab"; import { useActivityDateRange } from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange"; +import { BudgetPeriodTooltip } from "@/components/shared/BudgetPeriodTooltip"; interface UserInfoViewProps { userId: string; @@ -507,7 +508,15 @@ export default function UserInfoView({
-

Spend

+

+ Spend + +

${formatNumberWithCommas(userData.spend || 0, 2)}

diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx index 770315af49f..202a111ea97 100644 --- a/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/DashboardHeader.test.tsx @@ -124,7 +124,8 @@ describe("DashboardHeader breadcrumb", () => { const tools = await screen.findByRole("dialog", { name: "Gateway tools" }); expect(within(tools).getByRole("button", { name: "Switch workspace, current: AI Gateway" })).toBeInTheDocument(); expect(within(tools).getByRole("link", { name: "Docs" })).toBeInTheDocument(); - expect(within(tools).getByRole("button", { name: "Aria Agent" })).toBeInTheDocument(); + expect(within(screen.getByRole("banner")).getByRole("button", { name: "Aria Agent" })).toBeInTheDocument(); + expect(screen.getAllByRole("button", { name: "Aria Agent" })).toHaveLength(1); }); it("closes mobile tools when switching to desktop and keeps them closed when returning", async () => { diff --git a/ui/litellm-dashboard/src/components/DashboardHeader.tsx b/ui/litellm-dashboard/src/components/DashboardHeader.tsx index b31fc794f79..55f163c2495 100644 --- a/ui/litellm-dashboard/src/components/DashboardHeader.tsx +++ b/ui/litellm-dashboard/src/components/DashboardHeader.tsx @@ -73,43 +73,46 @@ export function DashboardHeader({ -

- {showWorkerSwitch && ( - <> - - - - )} +
- - - {!hideCommunityLinks && } - - - {deploymentHealth} - -
-
- - {deploymentHealth} - - - }> - - - - Gateway tools - - {showWorkerSwitch && } -
- - - - - {!hideCommunityLinks && } -
- - +
+ {showWorkerSwitch && ( + <> + + + + )} + + + {!hideCommunityLinks && } + + + {deploymentHealth} + +
+
+ + {deploymentHealth} + + + } + > + + + + Gateway tools + + {showWorkerSwitch && } +
+ + + + {!hideCommunityLinks && } +
+ + +
); diff --git a/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx b/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx index 70fe23202bf..085334567ed 100644 --- a/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx +++ b/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx @@ -34,7 +34,7 @@ import type { Tag } from "./tag_management/types"; import { ModelTeamSelect } from "./view_model/ModelTeamSelect"; import VectorStoreSelector from "./vector_store_management/VectorStoreSelector"; import { formatPtuUtcDisplay, utcIsoToPickerValue } from "../utils/ptuDatetime"; -import { isMaskedSecret } from "../utils/maskedSecretUtils"; +import { editableModelExtraParams, parseModelExtraParams } from "../utils/modelEditParams"; import { MAX_COST_PER_PTU_PER_HOUR, MAX_PTU_COUNT, @@ -127,22 +127,13 @@ const modelEditShape = { type ModelEditFieldName = keyof typeof modelEditShape; -const isJson = (value: string): boolean => { - try { - JSON.parse(value); - return true; - } catch { - return false; - } -}; - const buildSchema = (ptuEnabled: boolean, isFieldTouched: (field: TouchedPricingField) => boolean) => z.object(modelEditShape).superRefine((values, ctx) => { const reject = (path: ModelEditFieldName, message: string) => ctx.addIssue({ code: "custom", path: [path], message }); - if (values.litellm_extra_params && !isJson(values.litellm_extra_params)) { - reject("litellm_extra_params", "Please enter valid JSON"); + if (parseModelExtraParams(values.litellm_extra_params) === null) { + reject("litellm_extra_params", "Please enter a valid JSON object"); } // antd validates only mounted fields, and the PTU block does not render when the flag is off. @@ -245,15 +236,7 @@ export const toModelEditFormValues = (localModelData: any, isWildcardModel: bool // antd never mounted this field for a non-wildcard model, so the key must be absent, not null. ...(isWildcardModel ? { health_check_model: localModelData.model_info?.health_check_model } : {}), litellm_credential_name: localModelData.litellm_params?.litellm_credential_name ?? null, - litellm_extra_params: JSON.stringify( - Object.fromEntries( - Object.entries(localModelData.litellm_params || {}).filter( - ([key, value]) => key !== "litellm_credential_name" && !isMaskedSecret(value), - ), - ), - null, - 2, - ), + litellm_extra_params: JSON.stringify(editableModelExtraParams(localModelData.litellm_params ?? {}), null, 2), team_id: localModelData.model_info?.team_id ?? undefined, }); @@ -775,7 +758,7 @@ const ModelInfoEditForm: React.FC = ({ LiteLLM Params @@ -784,9 +767,10 @@ const ModelInfoEditForm: React.FC = ({ {({ value, ...control }) => (