diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index 1c6c318c717..b38c342e6d8 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -23,7 +23,9 @@ jobs: version: "0.10.9" - name: Update JSON Data run: | - uv run --frozen --with 'aiohttp==3.13.3' python ".github/workflows/auto_update_price_and_context_window_file.py" + uv run --frozen --with 'aiohttp==3.13.3' --with 'pydantic==2.13.4' python ".github/workflows/auto_update_price_and_context_window_file.py" + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - name: Create Pull Request run: | git add model_prices_and_context_window.json diff --git a/.github/workflows/auto_update_price_and_context_window_file.py b/.github/workflows/auto_update_price_and_context_window_file.py index 461d8d347d9..7f545cc396e 100644 --- a/.github/workflows/auto_update_price_and_context_window_file.py +++ b/.github/workflows/auto_update_price_and_context_window_file.py @@ -1,6 +1,10 @@ import asyncio +import os +import re import aiohttp import json +from pydantic import BaseModel + # Asynchronously fetch data from a given URL async def fetch_data(url): @@ -15,22 +19,24 @@ async def fetch_data(url): resp_json = await resp.json() print("Fetch the data from URL.") # Return the 'data' field from the JSON response - return resp_json['data'] + return resp_json["data"] except Exception as e: # Print an error message if fetching data fails print("Error fetching data from URL:", e) return None + # Synchronize local data with remote data def sync_local_data_with_remote(local_data, remote_data): # Update existing keys in local_data with values from remote_data - for key in (set(local_data) & set(remote_data)): + for key in set(local_data) & set(remote_data): local_data[key].update(remote_data[key]) # Add new keys from remote_data to local_data - for key in (set(remote_data) - set(local_data)): + for key in set(remote_data) - set(local_data): local_data[key] = remote_data[key] + # Write data to the json file def write_to_file(file_path, data): try: @@ -43,6 +49,7 @@ def write_to_file(file_path, data): # Print an error message if writing to file fails print("Error updating JSON file:", e) + # Update the existing models and add the missing models for OpenRouter def transform_openrouter_data(data): transformed = {} @@ -54,33 +61,41 @@ def transform_openrouter_data(data): } # Add 'max_output_tokens' as a field if it is not None - if "top_provider" in row and "max_completion_tokens" in row["top_provider"] and row["top_provider"]["max_completion_tokens"] is not None: - obj['max_output_tokens'] = int(row["top_provider"]["max_completion_tokens"]) + if ( + "top_provider" in row + and "max_completion_tokens" in row["top_provider"] + and row["top_provider"]["max_completion_tokens"] is not None + ): + obj["max_output_tokens"] = int(row["top_provider"]["max_completion_tokens"]) # Add the field 'output_cost_per_token' - obj.update({ - "output_cost_per_token": float(row["pricing"]["completion"]), - }) + obj.update( + { + "output_cost_per_token": float(row["pricing"]["completion"]), + } + ) # Add field 'input_cost_per_image' if it exists and is non-zero - if "pricing" in row and "image" in row["pricing"] and float(row["pricing"]["image"]) != 0.0: - obj['input_cost_per_image'] = float(row["pricing"]["image"]) + if ( + "pricing" in row + and "image" in row["pricing"] + and float(row["pricing"]["image"]) != 0.0 + ): + obj["input_cost_per_image"] = float(row["pricing"]["image"]) # Add the fields 'litellm_provider' and 'mode' - obj.update({ - "litellm_provider": "openrouter", - "mode": "chat" - }) + obj.update({"litellm_provider": "openrouter", "mode": "chat"}) # Add the 'supports_vision' field if the modality is 'multimodal' - if row.get('architecture', {}).get('modality') == 'multimodal': - obj['supports_vision'] = True + if row.get("architecture", {}).get("modality") == "multimodal": + obj["supports_vision"] = True # Use a composite key to store the transformed object - transformed[f'openrouter/{row["id"]}'] = obj + transformed[f"openrouter/{row['id']}"] = obj return transformed + # Update the existing models and add the missing models for Vercel AI Gateway def transform_vercel_ai_gateway_data(data): transformed = {} @@ -89,27 +104,181 @@ def transform_vercel_ai_gateway_data(data): "max_tokens": row["context_window"], "input_cost_per_token": float(row["pricing"]["input"]), "output_cost_per_token": float(row["pricing"]["output"]), - 'max_output_tokens': row['max_tokens'], - 'max_input_tokens': row["context_window"], + "max_output_tokens": row["max_tokens"], + "max_input_tokens": row["context_window"], } # Handle cache pricing if available if "pricing" in row: - if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None: - obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}") - - if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None: - obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}") + if ( + "input_cache_read" in row["pricing"] + and row["pricing"]["input_cache_read"] is not None + ): + obj["cache_read_input_token_cost"] = float( + f"{float(row['pricing']['input_cache_read']):e}" + ) + + if ( + "input_cache_write" in row["pricing"] + and row["pricing"]["input_cache_write"] is not None + ): + obj["cache_creation_input_token_cost"] = float( + f"{float(row['pricing']['input_cache_write']):e}" + ) mode = "embedding" if "embedding" in row["id"].lower() else "chat" - + obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode}) - transformed[f'vercel_ai_gateway/{row["id"]}'] = obj + transformed[f"vercel_ai_gateway/{row['id']}"] = obj return transformed +class _SupportFlag(BaseModel): + supported: bool = False + + +class _ThinkingTypes(BaseModel): + adaptive: _SupportFlag = _SupportFlag() + + +class _Thinking(BaseModel): + supported: bool = False + types: _ThinkingTypes = _ThinkingTypes() + + +class _Effort(BaseModel): + supported: bool = False + xhigh: _SupportFlag = _SupportFlag() + max: _SupportFlag = _SupportFlag() + + +class _Capabilities(BaseModel): + image_input: _SupportFlag = _SupportFlag() + pdf_input: _SupportFlag = _SupportFlag() + structured_outputs: _SupportFlag = _SupportFlag() + thinking: _Thinking = _Thinking() + effort: _Effort = _Effort() + + +class AnthropicModel(BaseModel): + id: str + max_input_tokens: int | None = None + max_tokens: int | None = None + capabilities: _Capabilities = _Capabilities() + + +class OpenRouterPricing(BaseModel): + prompt: float + completion: float + input_cache_read: float | None = None + input_cache_write: float | None = None + + +class OpenRouterModel(BaseModel): + id: str + pricing: OpenRouterPricing + + +class AnthropicEntry(BaseModel): + litellm_provider: str + mode: str + max_tokens: int | None = None + max_input_tokens: int | None = None + max_output_tokens: int | None = None + input_cost_per_token: float + output_cost_per_token: float + cache_read_input_token_cost: float | None = None + cache_creation_input_token_cost: float | None = None + supports_vision: bool | None = None + supports_pdf_input: bool | None = None + supports_response_schema: bool | None = None + supports_reasoning: bool | None = None + supports_adaptive_thinking: bool | None = None + supports_xhigh_reasoning_effort: bool | None = None + supports_max_reasoning_effort: bool | None = None + supports_function_calling: bool | None = None + supports_tool_choice: bool | None = None + supports_prompt_caching: bool | None = None + + +# Normalize a provider model id to a comparable key (drop prefix/suffix, dates, dot vs dash) +def canonical_model_id(model_id): + base = model_id.split("/")[-1].split(":")[0].lower() + base = re.sub(r"-\d{8}$", "", base) + return base.replace(".", "-") + + +# Build a price lookup from OpenRouter's Anthropic models, preferring base ids over ":variant" ids +def build_anthropic_price_index(openrouter_rows): + anthropic_rows = ( + OpenRouterModel.model_validate(row) + for row in openrouter_rows + if str(row.get("id", "")).startswith("anthropic/") + ) + return { + canonical_model_id(model.id): model.pricing + for model in sorted(anthropic_rows, key=lambda model: ":" not in model.id) + } + + +# Build native Anthropic entries for models that are new and have a known price +def transform_anthropic_data(anthropic_rows, price_index, existing_keys): + entries = {} + for row in anthropic_rows: + model = AnthropicModel.model_validate(row) + if model.id in existing_keys: + continue + pricing = price_index.get(canonical_model_id(model.id)) + if pricing is None: + print(f"Skipping {model.id}: no OpenRouter price match") + continue + caps = model.capabilities + entry = AnthropicEntry( + litellm_provider="anthropic", + mode="chat", + max_tokens=model.max_tokens, + max_input_tokens=model.max_input_tokens, + max_output_tokens=model.max_tokens, + input_cost_per_token=pricing.prompt, + output_cost_per_token=pricing.completion, + cache_read_input_token_cost=pricing.input_cache_read, + cache_creation_input_token_cost=pricing.input_cache_write, + supports_vision=caps.image_input.supported or None, + supports_pdf_input=caps.pdf_input.supported or None, + supports_response_schema=caps.structured_outputs.supported or None, + supports_reasoning=caps.thinking.supported or None, + supports_adaptive_thinking=caps.thinking.types.adaptive.supported or None, + supports_xhigh_reasoning_effort=caps.effort.xhigh.supported or None, + supports_max_reasoning_effort=caps.effort.max.supported or None, + supports_function_calling=True, + supports_tool_choice=True, + supports_prompt_caching=(pricing.input_cache_read is not None) or None, + ) + entries[model.id] = entry.model_dump(exclude_none=True) + return entries + + +# Fetch the Anthropic models list (authoritative for existence, context window, capabilities) +async def fetch_anthropic_models(api_key): + headers = {"x-api-key": api_key, "anthropic-version": "2023-06-01"} + url = "https://api.anthropic.com/v1/models?limit=1000" + try: + async with aiohttp.ClientSession() as session: + async with session.get(url, headers=headers) as resp: + resp.raise_for_status() + payload = await resp.json() + if payload.get("has_more"): + print( + "Warning: Anthropic models response truncated; pagination not implemented" + ) + return payload["data"] + except Exception as e: + print("Error fetching Anthropic models:", e) + return None + + # Load local data from a specified file def load_local_data(file_path): try: @@ -126,33 +295,53 @@ def load_local_data(file_path): print("Error decoding JSON:", e) return None + def main(): - local_file_path = "model_prices_and_context_window.json" # Path to the local data file - openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data - vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data + local_file_path = ( + "model_prices_and_context_window.json" # Path to the local data file + ) + openrouter_url = ( + "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data + ) + vercel_ai_gateway_url = ( + "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data + ) - # Load local data from file local_data = load_local_data(local_file_path) - - # Fetch OpenRouter data - openrouter_data = asyncio.run(fetch_data(openrouter_url)) - # Transform the fetched OpenRouter data - openrouter_data = transform_openrouter_data(openrouter_data) - - # Fetch Vercel AI Gateway data - vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url)) - # Transform the fetched Vercel AI Gateway data - vercel_data = transform_vercel_ai_gateway_data(vercel_data) - - # Combine both datasets - all_remote_data = {**openrouter_data, **vercel_data} + if not local_data: + print("Failed to load local model data.") + return + existing_keys = frozenset(local_data) + + openrouter_rows = asyncio.run(fetch_data(openrouter_url)) + vercel_rows = asyncio.run(fetch_data(vercel_ai_gateway_url)) + + remote_data = {} + if openrouter_rows: + remote_data.update(transform_openrouter_data(openrouter_rows)) + if vercel_rows: + remote_data.update(transform_vercel_ai_gateway_data(vercel_rows)) + + if not remote_data: + print("Failed to fetch model data from remote URLs.") + return + + sync_local_data_with_remote(local_data, remote_data) + + anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY") + if anthropic_api_key and openrouter_rows: + anthropic_rows = asyncio.run(fetch_anthropic_models(anthropic_api_key)) + if anthropic_rows: + price_index = build_anthropic_price_index(openrouter_rows) + for key, entry in transform_anthropic_data( + anthropic_rows, price_index, existing_keys + ).items(): + local_data.setdefault(key, entry) + elif not anthropic_api_key: + print("ANTHROPIC_API_KEY not set; skipping native Anthropic source.") + + write_to_file(local_file_path, local_data) - # If both local and openrouter data are available, synchronize and save - if local_data and all_remote_data: - sync_local_data_with_remote(local_data, all_remote_data) - write_to_file(local_file_path, local_data) - else: - print("Failed to fetch model data from either local file or URL.") # Entry point of the script if __name__ == "__main__": diff --git a/tests/test_litellm/test_auto_update_price_and_context_window_file.py b/tests/test_litellm/test_auto_update_price_and_context_window_file.py new file mode 100644 index 00000000000..ea1caa8cc18 --- /dev/null +++ b/tests/test_litellm/test_auto_update_price_and_context_window_file.py @@ -0,0 +1,137 @@ +import importlib.util +from pathlib import Path + +_MODULE_PATH = ( + Path(__file__).resolve().parents[2] + / ".github" + / "workflows" + / "auto_update_price_and_context_window_file.py" +) + + +def _load_module(): + spec = importlib.util.spec_from_file_location( + "auto_update_price_and_context_window_file", _MODULE_PATH + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +mod = _load_module() + + +def _opus_model(model_id="claude-opus-4-9"): + return { + "id": model_id, + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "image_input": {"supported": True}, + "pdf_input": {"supported": True}, + "structured_outputs": {"supported": True}, + "thinking": {"supported": True, "types": {"adaptive": {"supported": True}}}, + "effort": { + "supported": True, + "xhigh": {"supported": True}, + "max": {"supported": True}, + }, + }, + } + + +def _openrouter_rows(): + return [ + { + "id": "anthropic/claude-opus-4.9", + "pricing": { + "prompt": "0.000005", + "completion": "0.000025", + "input_cache_read": "0.0000005", + "input_cache_write": "0.00000625", + }, + }, + { + "id": "openai/gpt-5.5", + "pricing": {"prompt": "0.000005", "completion": "0.00003"}, + }, + ] + + +def test_canonical_id_normalizes_dots_dates_and_prefix(): + assert mod.canonical_model_id("anthropic/claude-opus-4.9") == "claude-opus-4-9" + assert mod.canonical_model_id("claude-opus-4-5-20251101") == "claude-opus-4-5" + assert ( + mod.canonical_model_id("anthropic/claude-opus-4.9:thinking") + == "claude-opus-4-9" + ) + + +def test_price_index_prefers_base_over_variant(): + rows = [ + { + "id": "anthropic/claude-opus-4.9:thinking", + "pricing": {"prompt": "9", "completion": "9"}, + }, + { + "id": "anthropic/claude-opus-4.9", + "pricing": {"prompt": "0.000005", "completion": "0.000025"}, + }, + ] + index = mod.build_anthropic_price_index(rows) + assert index["claude-opus-4-9"].prompt == 0.000005 + assert index["claude-opus-4-9"].completion == 0.000025 + + +def test_new_model_gets_price_context_and_capabilities(): + index = mod.build_anthropic_price_index(_openrouter_rows()) + entries = mod.transform_anthropic_data([_opus_model()], index, frozenset()) + + assert "claude-opus-4-9" in entries + entry = entries["claude-opus-4-9"] + assert entry["litellm_provider"] == "anthropic" + assert entry["mode"] == "chat" + assert entry["max_input_tokens"] == 1000000 + assert entry["max_output_tokens"] == 128000 + assert entry["max_tokens"] == 128000 + assert entry["input_cost_per_token"] == 0.000005 + assert entry["output_cost_per_token"] == 0.000025 + assert entry["cache_read_input_token_cost"] == 0.0000005 + assert entry["cache_creation_input_token_cost"] == 0.00000625 + assert entry["supports_vision"] is True + assert entry["supports_pdf_input"] is True + assert entry["supports_reasoning"] is True + assert entry["supports_adaptive_thinking"] is True + assert entry["supports_response_schema"] is True + assert entry["supports_xhigh_reasoning_effort"] is True + assert entry["supports_max_reasoning_effort"] is True + assert entry["supports_function_calling"] is True + assert entry["supports_tool_choice"] is True + assert entry["supports_prompt_caching"] is True + + +def test_model_without_price_match_is_skipped(): + index = mod.build_anthropic_price_index(_openrouter_rows()) + entries = mod.transform_anthropic_data( + [_opus_model("claude-unlisted-model")], index, frozenset() + ) + assert entries == {} + + +def test_existing_curated_model_is_not_re_added(): + index = mod.build_anthropic_price_index(_openrouter_rows()) + entries = mod.transform_anthropic_data( + [_opus_model()], index, frozenset({"claude-opus-4-9"}) + ) + assert entries == {} + + +def test_false_capabilities_are_omitted(): + model = _opus_model() + model["capabilities"]["image_input"]["supported"] = False + model["capabilities"]["effort"]["xhigh"]["supported"] = False + index = mod.build_anthropic_price_index(_openrouter_rows()) + entry = mod.transform_anthropic_data([model], index, frozenset())["claude-opus-4-9"] + assert "supports_vision" not in entry + assert "supports_xhigh_reasoning_effort" not in entry + assert entry["supports_pdf_input"] is True