diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index 1a638a4a331..d391c0bd6ce 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -24,9 +24,12 @@ jobs: - name: Update JSON Data run: | uv run --frozen --with 'aiohttp==3.13.3' python ".github/workflows/auto_update_price_and_context_window_file.py" + - name: Regenerate JSON Schema + run: | + uv run --frozen python ci_cd/generate_model_prices_schema.py - name: Create Pull Request run: | - git add model_prices_and_context_window.json + git add model_prices_and_context_window.json model_prices_and_context_window.schema.json git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')" gh pr create --title "Update model_prices_and_context_window.json file" \ --body "Automated update for model_prices_and_context_window.json" \ diff --git a/.github/workflows/test-model-map.yaml b/.github/workflows/test-model-map.yaml index b2170d9f6a4..cf4b0eb21a1 100644 --- a/.github/workflows/test-model-map.yaml +++ b/.github/workflows/test-model-map.yaml @@ -22,3 +22,12 @@ jobs: - name: Validate model_prices_and_context_window.json run: | jq empty model_prices_and_context_window.json + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Check model_prices_and_context_window.schema.json is in sync + run: | + uv run --frozen python ci_cd/generate_model_prices_schema.py --check diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py new file mode 100644 index 00000000000..83c48f788f8 --- /dev/null +++ b/ci_cd/generate_model_prices_schema.py @@ -0,0 +1,322 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Optional + +import jsonschema + +REPO_ROOT = Path(__file__).parent.parent +PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" +SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json" + +SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"}) + +JsonSchema = dict + +NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0} +NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0} +BOOLEAN: JsonSchema = {"type": "boolean"} +STRING: JsonSchema = {"type": "string"} + +EXTRA_BOOLEAN_KEYS = frozenset( + { + "gemini_native_audio", + "gemini_audio_only_live", + "uses_embed_content", + "use_openai_responses_path", + "bedrock_converse_supports_strict_tools", + } +) + +OBJECT_KEYS: dict[str, JsonSchema] = { + "search_context_cost_per_query": { + "type": "object", + "description": "USD cost per web search query, keyed by search context size.", + "properties": { + "search_context_size_low": NONNEG_NUMBER, + "search_context_size_medium": NONNEG_NUMBER, + "search_context_size_high": NONNEG_NUMBER, + }, + "additionalProperties": False, + }, + "metadata": { + "type": "object", + "description": "Free-form notes about the entry (e.g. pricing derivation).", + }, + "provider_specific_entry": { + "type": "object", + "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).", + }, +} + +ARRAY_KEYS: dict[str, JsonSchema] = { + "supported_endpoints": { + "type": "array", + "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.", + "items": STRING, + }, + "supported_modalities": { + "type": "array", + "description": "Input modalities the model accepts.", + "items": {"type": "string", "enum": ["text", "image", "audio", "video"]}, + }, + "supported_output_modalities": { + "type": "array", + "description": "Output modalities the model can produce.", + "items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]}, + }, + "supported_regions": { + "type": "array", + "description": "Cloud regions the model is available in ('global' or region ids).", + "items": STRING, + }, + "tiered_pricing": { + "type": "array", + "description": "Context-length or result-count tiered rates; each tier's costs apply within its range.", + "items": { + "type": "object", + "properties": { + "range": { + "type": "array", + "description": "[min, max] prompt-token span this tier applies to.", + "items": NONNEG_NUMBER, + "minItems": 2, + "maxItems": 2, + }, + "max_results_range": { + "type": "array", + "description": "[min, max] result-count span this tier applies to (search models).", + "items": NONNEG_NUMBER, + "minItems": 2, + "maxItems": 2, + }, + "input_cost_per_token": NONNEG_NUMBER, + "output_cost_per_token": NONNEG_NUMBER, + "output_cost_per_reasoning_token": NONNEG_NUMBER, + "cache_read_input_token_cost": NONNEG_NUMBER, + "input_cost_per_query": NONNEG_NUMBER, + }, + "additionalProperties": False, + }, + }, +} + +INTEGER_KEYS: dict[str, JsonSchema] = { + "max_tokens": { + **NONNEG_INTEGER, + "description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.", + }, + "max_input_tokens": { + **NONNEG_INTEGER, + "description": "Maximum prompt/context tokens the model accepts.", + }, + "max_output_tokens": { + **NONNEG_INTEGER, + "description": "Maximum tokens the model can generate in one response.", + }, + "output_vector_size": { + **NONNEG_INTEGER, + "description": "Embedding dimension for embedding models.", + }, + "prompt_cache_min_tokens": { + **NONNEG_INTEGER, + "description": "Smallest prefix the provider will actually cache; absent means the provider default applies.", + }, + "tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."}, + "rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."}, +} + +NUMBER_KEYS: dict[str, JsonSchema] = { + "regional_processing_uplift_multiplier_eu": { + "type": "number", + "minimum": 1, + "description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).", + }, + "regional_processing_uplift_multiplier_us": { + "type": "number", + "minimum": 1, + "description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).", + }, +} + +COST_DESCRIPTIONS: dict[str, str] = { + "input_cost_per_token": "USD per prompt token.", + "output_cost_per_token": "USD per generated token.", + "output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.", + "cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.", + "cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.", + "input_cost_per_token_batches": "USD per prompt token via the provider's batch API.", + "output_cost_per_token_batches": "USD per generated token via the provider's batch API.", +} + + +def cost_description(key: str) -> Optional[str]: + if key in COST_DESCRIPTIONS: + return COST_DESCRIPTIONS[key] + if key.endswith("_flex"): + return "Flex service-tier rate for the same-named base field." + if key.endswith("_priority"): + return "Priority service-tier rate for the same-named base field." + if "_above_" in key: + return "Rate applied once the prompt exceeds the token threshold in the field name." + return None + + +def cost_schema(key: str) -> JsonSchema: + description = cost_description(key) + return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER) + + +def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]: + return { + "litellm_provider": { + "type": "string", + "description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.", + }, + "mode": { + "type": "string", + "description": "Primary API surface / task type of the model.", + "enum": list(modes), + }, + "source": { + "type": "string", + "description": "URL of the provider pricing/model page this entry was taken from.", + }, + "deprecation_date": { + "type": "string", + "description": "Date the provider deprecates the model, YYYY-MM-DD.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$", + }, + "web_search_billing_unit": { + "type": "string", + "description": "Whether web search is billed per query or per prompt.", + "enum": ["per_query", "per_prompt"], + }, + "bedrock_output_config_effort_ceiling": { + "type": "string", + "description": "Highest reasoning effort the Bedrock output_config accepts for this model.", + "enum": ["low", "medium", "high", "max", "xhigh"], + }, + "comment": STRING, + "audio_transcription_config": STRING, + } + + +def classify(key: str, modes: tuple) -> Optional[JsonSchema]: + curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS} + if key in curated: + return curated[key] + if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS: + return BOOLEAN + if "cost" in key: + return cost_schema(key) + return None + + +def build_schema(prices: dict) -> JsonSchema: + entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS} + all_keys = tuple(sorted({key for entry in entries.values() for key in entry})) + modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry})) + unclassified = tuple(key for key in all_keys if classify(key, modes) is None) + if unclassified: + raise SystemExit( + f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. " + f"Add them to the key tables in {Path(__file__).name} and rerun it." + ) + entry_properties = {key: classify(key, modes) for key in all_keys} + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "LiteLLM model_prices_and_context_window.json", + "description": ( + "Schema for LiteLLM's model price and context window registry " + "(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). " + "Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, " + "optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. " + "All costs are USD per unit. New optional fields are added regularly, so consumers should " + "ignore unknown fields rather than reject them." + ), + "type": "object", + "properties": { + "sample_spec": { + "type": "object", + "description": ( + "Documentation placeholder illustrating the entry shape; not a real model and not " + "schema-conformant (several values are prose)." + ), + }, + "fallback_generalizations": { + "type": "object", + "description": "Regex rules that generalize unknown model ids to known families; not a model entry.", + "properties": { + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": STRING, + "pattern": STRING, + "description": STRING, + }, + "required": ["name", "pattern"], + "additionalProperties": True, + }, + } + }, + "additionalProperties": False, + }, + }, + "additionalProperties": {"$ref": "#/$defs/modelEntry"}, + "$defs": { + "modelEntry": { + "type": "object", + "description": ( + "Pricing, limits, and capability flags for one model. Fields other than litellm_provider " + "are optional; boolean capability flags are simply omitted when unknown or false." + ), + "required": ["litellm_provider"], + "properties": entry_properties, + "additionalProperties": True, + } + }, + } + + +def render(schema: JsonSchema) -> str: + return json.dumps(schema, indent=2) + "\n" + + +def validation_errors(prices: dict, schema: JsonSchema) -> tuple: + validator = jsonschema.Draft202012Validator(schema) + return tuple( + f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}" + for error in validator.iter_errors(prices) + ) + + +def main() -> int: + check = "--check" in sys.argv[1:] + prices = json.loads(PRICES_PATH.read_text()) + rendered = render(build_schema(prices)) + errors = validation_errors(prices, json.loads(rendered)) + if errors: + print(f"{PRICES_PATH.name} does not validate against the generated schema:") + print("\n".join(errors[:20])) + return 1 + if not check: + SCHEMA_PATH.write_text(rendered) + print(f"wrote {SCHEMA_PATH}") + return 0 + if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered: + print( + f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. " + f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result." + ) + return 1 + print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json new file mode 100644 index 00000000000..6e3c3620232 --- /dev/null +++ b/model_prices_and_context_window.schema.json @@ -0,0 +1,741 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "LiteLLM model_prices_and_context_window.json", + "description": "Schema for LiteLLM's model price and context window registry (https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. All costs are USD per unit. New optional fields are added regularly, so consumers should ignore unknown fields rather than reject them.", + "type": "object", + "properties": { + "sample_spec": { + "type": "object", + "description": "Documentation placeholder illustrating the entry shape; not a real model and not schema-conformant (several values are prose)." + }, + "fallback_generalizations": { + "type": "object", + "description": "Regex rules that generalize unknown model ids to known families; not a model entry.", + "properties": { + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "pattern": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "required": [ + "name", + "pattern" + ], + "additionalProperties": true + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": { + "$ref": "#/$defs/modelEntry" + }, + "$defs": { + "modelEntry": { + "type": "object", + "description": "Pricing, limits, and capability flags for one model. Fields other than litellm_provider are optional; boolean capability flags are simply omitted when unknown or false.", + "required": [ + "litellm_provider" + ], + "properties": { + "annotation_cost_per_page": { + "type": "number", + "minimum": 0 + }, + "audio_transcription_config": { + "type": "string" + }, + "bedrock_converse_supports_strict_tools": { + "type": "boolean" + }, + "bedrock_output_config_effort_ceiling": { + "type": "string", + "description": "Highest reasoning effort the Bedrock output_config accepts for this model.", + "enum": [ + "low", + "medium", + "high", + "max", + "xhigh" + ] + }, + "cache_creation_input_audio_token_cost": { + "type": "number", + "minimum": 0 + }, + "cache_creation_input_token_cost": { + "type": "number", + "minimum": 0, + "description": "USD per token written to the provider's prompt cache." + }, + "cache_creation_input_token_cost_above_1hr": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "cache_creation_input_token_cost_above_200k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "cache_creation_input_token_cost_above_272k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "cache_creation_input_token_cost_flex": { + "type": "number", + "minimum": 0, + "description": "Flex service-tier rate for the same-named base field." + }, + "cache_creation_input_token_cost_priority": { + "type": "number", + "minimum": 0, + "description": "Priority service-tier rate for the same-named base field." + }, + "cache_read_input_audio_token_cost": { + "type": "number", + "minimum": 0 + }, + "cache_read_input_token_cost": { + "type": "number", + "minimum": 0, + "description": "USD per prompt token served from the provider's prompt cache." + }, + "cache_read_input_token_cost_above_200k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "cache_read_input_token_cost_above_200k_tokens_priority": { + "type": "number", + "minimum": 0, + "description": "Priority service-tier rate for the same-named base field." + }, + "cache_read_input_token_cost_above_272k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "cache_read_input_token_cost_above_272k_tokens_priority": { + "type": "number", + "minimum": 0, + "description": "Priority service-tier rate for the same-named base field." + }, + "cache_read_input_token_cost_above_512k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "cache_read_input_token_cost_flex": { + "type": "number", + "minimum": 0, + "description": "Flex service-tier rate for the same-named base field." + }, + "cache_read_input_token_cost_priority": { + "type": "number", + "minimum": 0, + "description": "Priority service-tier rate for the same-named base field." + }, + "citation_cost_per_token": { + "type": "number", + "minimum": 0 + }, + "code_interpreter_cost_per_session": { + "type": "number", + "minimum": 0 + }, + "comment": { + "type": "string" + }, + "deprecation_date": { + "type": "string", + "description": "Date the provider deprecates the model, YYYY-MM-DD.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}$" + }, + "gemini_audio_only_live": { + "type": "boolean" + }, + "gemini_native_audio": { + "type": "boolean" + }, + "input_cost_per_audio_per_second": { + "type": "number", + "minimum": 0 + }, + "input_cost_per_audio_per_second_above_128k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "input_cost_per_audio_token": { + "type": "number", + "minimum": 0 + }, + "input_cost_per_audio_token_priority": { + "type": "number", + "minimum": 0, + "description": "Priority service-tier rate for the same-named base field." + }, + "input_cost_per_character": { + "type": "number", + "minimum": 0 + }, + "input_cost_per_character_above_128k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "input_cost_per_image": { + "type": "number", + "minimum": 0 + }, + "input_cost_per_image_above_128k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "input_cost_per_image_token": { + "type": "number", + "minimum": 0 + }, + "input_cost_per_pixel": { + "type": "number", + "minimum": 0 + }, + "input_cost_per_query": { + "type": "number", + "minimum": 0 + }, + "input_cost_per_request": { + "type": "number", + "minimum": 0 + }, + "input_cost_per_second": { + "type": "number", + "minimum": 0 + }, + "input_cost_per_token": { + "type": "number", + "minimum": 0, + "description": "USD per prompt token." + }, + "input_cost_per_token_above_128k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "input_cost_per_token_above_200k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "input_cost_per_token_above_200k_tokens_priority": { + "type": "number", + "minimum": 0, + "description": "Priority service-tier rate for the same-named base field." + }, + "input_cost_per_token_above_256k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "input_cost_per_token_above_272k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "input_cost_per_token_above_272k_tokens_priority": { + "type": "number", + "minimum": 0, + "description": "Priority service-tier rate for the same-named base field." + }, + "input_cost_per_token_above_512k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "input_cost_per_token_batches": { + "type": "number", + "minimum": 0, + "description": "USD per prompt token via the provider's batch API." + }, + "input_cost_per_token_cache_hit": { + "type": "number", + "minimum": 0 + }, + "input_cost_per_token_flex": { + "type": "number", + "minimum": 0, + "description": "Flex service-tier rate for the same-named base field." + }, + "input_cost_per_token_priority": { + "type": "number", + "minimum": 0, + "description": "Priority service-tier rate for the same-named base field." + }, + "input_cost_per_video_per_second": { + "type": "number", + "minimum": 0 + }, + "input_cost_per_video_per_second_above_128k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "input_cost_per_video_per_second_above_15s_interval": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "input_cost_per_video_per_second_above_8s_interval": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "input_dbu_cost_per_token": { + "type": "number", + "minimum": 0 + }, + "litellm_provider": { + "type": "string", + "description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers." + }, + "max_input_tokens": { + "type": "integer", + "minimum": 0, + "description": "Maximum prompt/context tokens the model accepts." + }, + "max_output_tokens": { + "type": "integer", + "minimum": 0, + "description": "Maximum tokens the model can generate in one response." + }, + "max_tokens": { + "type": "integer", + "minimum": 0, + "description": "Legacy field: max output tokens if the provider specifies it, else max input tokens." + }, + "metadata": { + "type": "object", + "description": "Free-form notes about the entry (e.g. pricing derivation)." + }, + "mode": { + "type": "string", + "description": "Primary API surface / task type of the model.", + "enum": [ + "audio_speech", + "audio_transcription", + "chat", + "completion", + "embedding", + "image_edit", + "image_generation", + "moderation", + "ocr", + "realtime", + "rerank", + "responses", + "search", + "vector_store", + "video_generation" + ] + }, + "ocr_cost_per_credit": { + "type": "number", + "minimum": 0 + }, + "ocr_cost_per_page": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_audio_token": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_character": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_character_above_128k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "output_cost_per_image": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_image_token": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_pixel": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_reasoning_token": { + "type": "number", + "minimum": 0, + "description": "USD per reasoning/thinking token, when billed separately." + }, + "output_cost_per_second": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_second_1080p": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_token": { + "type": "number", + "minimum": 0, + "description": "USD per generated token." + }, + "output_cost_per_token_above_128k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "output_cost_per_token_above_200k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "output_cost_per_token_above_200k_tokens_priority": { + "type": "number", + "minimum": 0, + "description": "Priority service-tier rate for the same-named base field." + }, + "output_cost_per_token_above_256k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "output_cost_per_token_above_272k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "output_cost_per_token_above_272k_tokens_priority": { + "type": "number", + "minimum": 0, + "description": "Priority service-tier rate for the same-named base field." + }, + "output_cost_per_token_above_512k_tokens": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, + "output_cost_per_token_batches": { + "type": "number", + "minimum": 0, + "description": "USD per generated token via the provider's batch API." + }, + "output_cost_per_token_flex": { + "type": "number", + "minimum": 0, + "description": "Flex service-tier rate for the same-named base field." + }, + "output_cost_per_token_priority": { + "type": "number", + "minimum": 0, + "description": "Priority service-tier rate for the same-named base field." + }, + "output_cost_per_video_per_second": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_video_token": { + "type": "number", + "minimum": 0 + }, + "output_dbu_cost_per_token": { + "type": "number", + "minimum": 0 + }, + "output_vector_size": { + "type": "integer", + "minimum": 0, + "description": "Embedding dimension for embedding models." + }, + "prompt_cache_min_tokens": { + "type": "integer", + "minimum": 0, + "description": "Smallest prefix the provider will actually cache; absent means the provider default applies." + }, + "provider_specific_entry": { + "type": "object", + "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema)." + }, + "regional_processing_uplift_multiplier_eu": { + "type": "number", + "minimum": 1, + "description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%)." + }, + "regional_processing_uplift_multiplier_us": { + "type": "number", + "minimum": 1, + "description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%)." + }, + "rpm": { + "type": "integer", + "minimum": 0, + "description": "Provider default requests-per-minute limit." + }, + "search_context_cost_per_query": { + "type": "object", + "description": "USD cost per web search query, keyed by search context size.", + "properties": { + "search_context_size_low": { + "type": "number", + "minimum": 0 + }, + "search_context_size_medium": { + "type": "number", + "minimum": 0 + }, + "search_context_size_high": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "source": { + "type": "string", + "description": "URL of the provider pricing/model page this entry was taken from." + }, + "supported_endpoints": { + "type": "array", + "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.", + "items": { + "type": "string" + } + }, + "supported_modalities": { + "type": "array", + "description": "Input modalities the model accepts.", + "items": { + "type": "string", + "enum": [ + "text", + "image", + "audio", + "video" + ] + } + }, + "supported_output_modalities": { + "type": "array", + "description": "Output modalities the model can produce.", + "items": { + "type": "string", + "enum": [ + "text", + "image", + "audio", + "video", + "code" + ] + } + }, + "supported_regions": { + "type": "array", + "description": "Cloud regions the model is available in ('global' or region ids).", + "items": { + "type": "string" + } + }, + "supports_adaptive_thinking": { + "type": "boolean" + }, + "supports_assistant_prefill": { + "type": "boolean" + }, + "supports_audio_input": { + "type": "boolean" + }, + "supports_audio_output": { + "type": "boolean" + }, + "supports_computer_use": { + "type": "boolean" + }, + "supports_embedding_image_input": { + "type": "boolean" + }, + "supports_function_calling": { + "type": "boolean" + }, + "supports_image_input": { + "type": "boolean" + }, + "supports_image_size": { + "type": "boolean" + }, + "supports_low_reasoning_effort": { + "type": "boolean" + }, + "supports_max_reasoning_effort": { + "type": "boolean" + }, + "supports_mid_conversation_system": { + "type": "boolean" + }, + "supports_minimal_reasoning_effort": { + "type": "boolean" + }, + "supports_multimodal": { + "type": "boolean" + }, + "supports_native_streaming": { + "type": "boolean" + }, + "supports_native_structured_output": { + "type": "boolean" + }, + "supports_none_reasoning_effort": { + "type": "boolean" + }, + "supports_nova_canvas_image_edit": { + "type": "boolean" + }, + "supports_output_config": { + "type": "boolean" + }, + "supports_parallel_function_calling": { + "type": "boolean" + }, + "supports_parallel_tool_use_config": { + "type": "boolean" + }, + "supports_pdf_input": { + "type": "boolean" + }, + "supports_prompt_caching": { + "type": "boolean" + }, + "supports_reasoning": { + "type": "boolean" + }, + "supports_response_schema": { + "type": "boolean" + }, + "supports_sampling_params": { + "type": "boolean" + }, + "supports_speed": { + "type": "boolean" + }, + "supports_system_messages": { + "type": "boolean" + }, + "supports_tool_choice": { + "type": "boolean" + }, + "supports_url_context": { + "type": "boolean" + }, + "supports_video_input": { + "type": "boolean" + }, + "supports_vision": { + "type": "boolean" + }, + "supports_web_search": { + "type": "boolean" + }, + "supports_xhigh_reasoning_effort": { + "type": "boolean" + }, + "tiered_pricing": { + "type": "array", + "description": "Context-length or result-count tiered rates; each tier's costs apply within its range.", + "items": { + "type": "object", + "properties": { + "range": { + "type": "array", + "description": "[min, max] prompt-token span this tier applies to.", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 2, + "maxItems": 2 + }, + "max_results_range": { + "type": "array", + "description": "[min, max] result-count span this tier applies to (search models).", + "items": { + "type": "number", + "minimum": 0 + }, + "minItems": 2, + "maxItems": 2 + }, + "input_cost_per_token": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_token": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_reasoning_token": { + "type": "number", + "minimum": 0 + }, + "cache_read_input_token_cost": { + "type": "number", + "minimum": 0 + }, + "input_cost_per_query": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + } + }, + "tpm": { + "type": "integer", + "minimum": 0, + "description": "Provider default tokens-per-minute limit." + }, + "use_openai_responses_path": { + "type": "boolean" + }, + "uses_embed_content": { + "type": "boolean" + }, + "web_search_billing_unit": { + "type": "string", + "description": "Whether web search is billed per query or per prompt.", + "enum": [ + "per_query", + "per_prompt" + ] + } + }, + "additionalProperties": true + } + } +} diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py new file mode 100644 index 00000000000..95b9952ad18 --- /dev/null +++ b/tests/test_litellm/test_model_prices_schema.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import jsonschema +import pytest + +REPO_ROOT = Path(__file__).parents[2] +GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py" +PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" +SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json" + + +def load_generator(): + spec = importlib.util.spec_from_file_location("generate_model_prices_schema", GENERATOR_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def committed_schema() -> dict: + return json.loads(SCHEMA_PATH.read_text()) + + +@pytest.fixture(scope="module") +def prices() -> dict: + return json.loads(PRICES_PATH.read_text()) + + +def test_committed_schema_matches_generator_output(prices: dict, committed_schema: dict): + generator = load_generator() + regenerated = json.loads(generator.render(generator.build_schema(prices))) + assert regenerated == committed_schema, ( + "model_prices_and_context_window.schema.json is out of sync; " + "run `python ci_cd/generate_model_prices_schema.py` and commit the result" + ) + + +def test_prices_file_validates_against_committed_schema(prices: dict, committed_schema: dict): + validator = jsonschema.Draft202012Validator(committed_schema) + errors = [ + f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}" + for error in validator.iter_errors(prices) + ] + assert errors == [] + + +@pytest.mark.parametrize( + "entry", + [ + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": "0.01"}, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": -1}, + {"litellm_provider": "openai", "mode": "not_a_real_mode"}, + {"mode": "chat"}, + {"litellm_provider": "openai", "deprecation_date": "June 2026"}, + {"litellm_provider": "openai", "supported_modalities": ["smell"]}, + {"litellm_provider": "openai", "supports_vision": "yes"}, + {"litellm_provider": "openai", "max_tokens": 8191.5}, + {"litellm_provider": "openai", "tiered_pricing": [{"unknown_tier_field": 1}]}, + ], + ids=[ + "cost_as_string", + "negative_cost", + "unknown_mode", + "missing_provider", + "non_iso_deprecation_date", + "unknown_modality", + "boolean_flag_as_string", + "fractional_max_tokens", + "unknown_tiered_pricing_field", + ], +) +def test_schema_rejects_malformed_entries(committed_schema: dict, entry: dict): + validator = jsonschema.Draft202012Validator(committed_schema) + assert not validator.is_valid({"some-model": entry}) + + +def test_schema_accepts_minimal_and_unknown_optional_fields(committed_schema: dict): + validator = jsonschema.Draft202012Validator(committed_schema) + assert validator.is_valid({"some-model": {"litellm_provider": "openai"}}) + assert validator.is_valid({"some-model": {"litellm_provider": "openai", "brand_new_field": {"nested": True}}})