From 9c5db54d53ed1a8d77704c7c923534714dbd43c2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:22:05 -0700 Subject: [PATCH] fix(ci): sync openrouter and vercel_ai_gateway prices without crashing on unpriced catalog rows --- ...to_update_price_and_context_window_file.py | 124 +++++++++--------- .../auto_update_price_and_context_window.yml | 47 +++++-- ...to_update_price_and_context_window_file.py | 81 ++++++++++++ 3 files changed, 178 insertions(+), 74 deletions(-) create mode 100644 tests/test_litellm/test_auto_update_price_and_context_window_file.py diff --git a/.github/scripts/auto_update_price_and_context_window_file.py b/.github/scripts/auto_update_price_and_context_window_file.py index 56c3c7d547f..1b46aa17454 100644 --- a/.github/scripts/auto_update_price_and_context_window_file.py +++ b/.github/scripts/auto_update_price_and_context_window_file.py @@ -1,6 +1,34 @@ import asyncio -import aiohttp import json +from types import MappingProxyType +from typing import Final + +import aiohttp +from pydantic import BaseModel + +COST_MAP_PATHS: Final = ( + "model_prices_and_context_window.json", + "litellm/model_prices_and_context_window_backup.json", +) +OPENROUTER_URL: Final = "https://openrouter.ai/api/v1/models" +VERCEL_AI_GATEWAY_URL: Final = "https://ai-gateway.vercel.sh/v1/models" +VERCEL_TYPE_TO_MODE: Final = MappingProxyType({"language": "chat", "embedding": "embedding"}) + + +class VercelPricing(BaseModel): + input: float | None = None + output: float | None = None + input_cache_read: float | None = None + input_cache_write: float | None = None + + +class VercelModel(BaseModel): + id: str + type: str + context_window: int | None = None + max_tokens: int | None = None + pricing: VercelPricing | None = None + # Asynchronously fetch data from a given URL async def fetch_data(url): @@ -31,17 +59,11 @@ def sync_local_data_with_remote(local_data, remote_data): for key in (set(remote_data) - set(local_data)): local_data[key] = remote_data[key] -# Write data to the json file -def write_to_file(file_path, data): - try: - # Open the file in write mode - with open(file_path, "w") as file: - # Dump the data as JSON into the file - json.dump(data, file, indent=4) - print("Values updated successfully.") - except Exception as e: - # Print an error message if writing to file fails - print("Error updating JSON file:", e) +def write_to_file(file_path: str, data: dict[str, object]) -> None: + with open(file_path, "w", encoding="utf-8") as file: + file.write(json.dumps(data, indent=4, ensure_ascii=False) + "\n") + print(f"Wrote {file_path}.") + # Update the existing models and add the missing models for OpenRouter def transform_openrouter_data(data): @@ -81,40 +103,30 @@ def transform_openrouter_data(data): return transformed -# Update the existing models and add the missing models for Vercel AI Gateway -def transform_vercel_ai_gateway_data(data): - transformed = {} - for row in data: - obj = {} +def _vercel_entry(model: VercelModel) -> dict[str, object] | None: + mode: Final = VERCEL_TYPE_TO_MODE[model.type] + pricing: Final = model.pricing if model.pricing is not None else VercelPricing() + if pricing.input is None or (mode == "chat" and pricing.output is None): + print(f"Skipping vercel_ai_gateway/{model.id}: the catalog lists no per-token price for it.") + return None + candidate: Final = { + "max_tokens": model.max_tokens, + "max_input_tokens": model.context_window, + "max_output_tokens": model.max_tokens, + "input_cost_per_token": pricing.input, + "output_cost_per_token": pricing.output if pricing.output is not None else 0.0, + "cache_read_input_token_cost": pricing.input_cache_read, + "cache_creation_input_token_cost": pricing.input_cache_write, + "litellm_provider": "vercel_ai_gateway", + "mode": mode, + } + return {key: value for key, value in candidate.items() if value is not None} - if "context_window" in row: - obj['max_tokens'] = row["context_window"] - obj['max_input_tokens'] = row["context_window"] - if "pricing" in row: - if "input" in row["pricing"] and row["pricing"]["input"] is not None: - obj['input_cost_per_token'] = float(row["pricing"]["input"]) - if "output" in row["pricing"] and row["pricing"]["output"] is not None: - obj['output_cost_per_token'] = float(row["pricing"]["output"]) - - if "max_tokens" in row: - obj['max_output_tokens'] = row['max_tokens'] - - # Handle cache pricing if available - if "pricing" in row: - if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None: - obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}") - - if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None: - obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}") - - mode = "embedding" if "embedding" in row["id"].lower() else "chat" - - obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode}) - - transformed[f'vercel_ai_gateway/{row["id"]}'] = obj - - return transformed +def transform_vercel_ai_gateway_data(data: list[dict[str, object]]) -> dict[str, dict[str, object]]: + models: Final = tuple(VercelModel.model_validate(row) for row in data if row.get("type") in VERCEL_TYPE_TO_MODE) + entries: Final = ((f"vercel_ai_gateway/{model.id}", _vercel_entry(model)) for model in models) + return {key: entry for key, entry in entries if entry is not None} # Load local data from a specified file @@ -134,30 +146,16 @@ def load_local_data(file_path): return None def main(): - local_file_path = "model_prices_and_context_window.json" # Path to the local data file - openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data - vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data + local_data = load_local_data(COST_MAP_PATHS[0]) - # Load local data from file - local_data = load_local_data(local_file_path) - - # Fetch OpenRouter data - openrouter_data = asyncio.run(fetch_data(openrouter_url)) - # Transform the fetched OpenRouter data - openrouter_data = transform_openrouter_data(openrouter_data) - - # Fetch Vercel AI Gateway data - vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url)) - # Transform the fetched Vercel AI Gateway data - vercel_data = transform_vercel_ai_gateway_data(vercel_data) - - # Combine both datasets + openrouter_data = transform_openrouter_data(asyncio.run(fetch_data(OPENROUTER_URL))) + vercel_data = transform_vercel_ai_gateway_data(asyncio.run(fetch_data(VERCEL_AI_GATEWAY_URL))) all_remote_data = {**openrouter_data, **vercel_data} - # If both local and openrouter data are available, synchronize and save if local_data and all_remote_data: sync_local_data_with_remote(local_data, all_remote_data) - write_to_file(local_file_path, local_data) + for path in COST_MAP_PATHS: + write_to_file(path, local_data) else: print("Failed to fetch model data from either local file or URL.") diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index 7e40a860ee9..9a4c8ee004b 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -2,8 +2,8 @@ name: Updates model_prices_and_context_window.json and Create Pull Request on: schedule: - - cron: "0 0 * * 0" # Run every Sundays at midnight - #- cron: "0 0 * * *" # Run daily at midnight + - cron: "0 0 * * 0" + workflow_dispatch: permissions: contents: write @@ -21,19 +21,44 @@ jobs: uses: ./.github/actions/setup-uv-with-retries with: version: "0.10.9" - - name: Update JSON Data + - name: Look for an already-open price map update PR + id: existing run: | - uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py" + open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName --jq '[.[].headRefName | select(startswith("litellm_price_map_auto_update_"))] | first // empty')" + if [ -n "$open_pr" ]; then + echo "Skipping the sync: the price map update PR on $open_pr is still open" + fi + echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }} + - name: Update JSON Data + if: steps.existing.outputs.open_pr == '' + run: | + uv run --frozen --with 'aiohttp==3.13.3' python .github/scripts/auto_update_price_and_context_window_file.py - name: Regenerate JSON Schema + if: steps.existing.outputs.open_pr == '' run: | uv run --frozen python ci_cd/generate_model_prices_schema.py - name: Create Pull Request + if: steps.existing.outputs.open_pr == '' run: | - git add model_prices_and_context_window.json model_prices_and_context_window.schema.json - git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')" - gh pr create --title "Update model_prices_and_context_window.json file" \ - --body "Automated update for model_prices_and_context_window.json" \ - --head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \ - --base main + if git diff --quiet; then + echo "Price map already in sync with the OpenRouter and Vercel AI Gateway catalogs; no PR needed." + exit 0 + fi + today="$(date +'%Y-%m-%d')" + branch="litellm_price_map_auto_update_$today" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -b "$branch" + git add model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json model_prices_and_context_window.schema.json + git commit -m "chore(models): sync openrouter and vercel_ai_gateway prices $today" + gh auth setup-git + git push origin "$branch" + gh pr create \ + --title "chore(models): sync openrouter and vercel_ai_gateway prices $today" \ + --body "Automated sync of model_prices_and_context_window.json from the OpenRouter and Vercel AI Gateway model catalogs" \ + --head "$branch" \ + --base litellm_internal_staging env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} + GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }} diff --git a/tests/test_litellm/test_auto_update_price_and_context_window_file.py b/tests/test_litellm/test_auto_update_price_and_context_window_file.py new file mode 100644 index 00000000000..b97f6954b91 --- /dev/null +++ b/tests/test_litellm/test_auto_update_price_and_context_window_file.py @@ -0,0 +1,81 @@ +import importlib.util +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / ".github" / "scripts" / "auto_update_price_and_context_window_file.py" + +_spec = importlib.util.spec_from_file_location("auto_update_price_and_context_window_file", SCRIPT) +assert _spec is not None and _spec.loader is not None +price_sync = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(price_sync) + +LANGUAGE_ROW = { + "id": "anthropic/claude-sonnet-4.5", + "type": "language", + "context_window": 200000, + "max_tokens": 64000, + "pricing": { + "input": "0.000003", + "output": "0.000015", + "input_cache_read": "0.0000003", + "input_cache_write": "0.00000375", + }, +} +EMBEDDING_ROW = { + "id": "google/gemini-embedding-001", + "type": "embedding", + "context_window": 2048, + "max_tokens": 0, + "pricing": {"input": "0.00000015"}, +} +UNPRICED_LANGUAGE_ROW = { + "id": "perplexity/sonar", + "type": "language", + "context_window": 128000, + "max_tokens": 8000, +} +VIDEO_ROW = { + "id": "google/veo-3.1", + "type": "video", + "pricing": {"video_duration_pricing": [{"resolution": "720p", "price_per_second": "0.4"}]}, +} +IMAGE_ROW = {"id": "openai/gpt-image-1", "type": "image"} + + +def test_vercel_rows_without_per_token_pricing_are_skipped_instead_of_crashing() -> None: + result = price_sync.transform_vercel_ai_gateway_data( + [LANGUAGE_ROW, EMBEDDING_ROW, UNPRICED_LANGUAGE_ROW, VIDEO_ROW, IMAGE_ROW] + ) + + assert result == { + "vercel_ai_gateway/anthropic/claude-sonnet-4.5": { + "max_tokens": 64000, + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost": 3.75e-06, + "litellm_provider": "vercel_ai_gateway", + "mode": "chat", + }, + "vercel_ai_gateway/google/gemini-embedding-001": { + "max_tokens": 0, + "max_input_tokens": 2048, + "max_output_tokens": 0, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 0.0, + "litellm_provider": "vercel_ai_gateway", + "mode": "embedding", + }, + } + + +def test_write_to_file_matches_the_checked_in_cost_map_format(tmp_path: Path) -> None: + target = tmp_path / "model_prices_and_context_window.json" + + price_sync.write_to_file(str(target), {"vercel_ai_gateway/x": {"mode": "chat", "input_cost_per_token": 1e-06}}) + + assert target.read_text(encoding="utf-8") == ( + '{\n "vercel_ai_gateway/x": {\n "mode": "chat",\n "input_cost_per_token": 1e-06\n }\n}\n' + )