mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
ci: publish a generated JSON schema for model_prices_and_context_window.json
This commit is contained in:
mateo-berri
parent
2b7e01bb7e
commit
4556dfa930
5 changed files with 1161 additions and 1 deletions
|
|
@ -24,9 +24,12 @@ jobs:
|
|||
- name: Update JSON Data
|
||||
run: |
|
||||
uv run --frozen --with 'aiohttp==3.13.3' python ".github/workflows/auto_update_price_and_context_window_file.py"
|
||||
- name: Regenerate JSON Schema
|
||||
run: |
|
||||
uv run --frozen python ci_cd/generate_model_prices_schema.py
|
||||
- name: Create Pull Request
|
||||
run: |
|
||||
git add model_prices_and_context_window.json
|
||||
git add model_prices_and_context_window.json model_prices_and_context_window.schema.json
|
||||
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
|
||||
gh pr create --title "Update model_prices_and_context_window.json file" \
|
||||
--body "Automated update for model_prices_and_context_window.json" \
|
||||
|
|
|
|||
9
.github/workflows/test-model-map.yaml
vendored
9
.github/workflows/test-model-map.yaml
vendored
|
|
@ -22,3 +22,12 @@ jobs:
|
|||
- name: Validate model_prices_and_context_window.json
|
||||
run: |
|
||||
jq empty model_prices_and_context_window.json
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Check model_prices_and_context_window.schema.json is in sync
|
||||
run: |
|
||||
uv run --frozen python ci_cd/generate_model_prices_schema.py --check
|
||||
|
|
|
|||
322
ci_cd/generate_model_prices_schema.py
Normal file
322
ci_cd/generate_model_prices_schema.py
Normal file
|
|
@ -0,0 +1,322 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import jsonschema
|
||||
|
||||
REPO_ROOT = Path(__file__).parent.parent
|
||||
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
|
||||
|
||||
SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"})
|
||||
|
||||
JsonSchema = dict
|
||||
|
||||
NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
|
||||
NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
|
||||
BOOLEAN: JsonSchema = {"type": "boolean"}
|
||||
STRING: JsonSchema = {"type": "string"}
|
||||
|
||||
EXTRA_BOOLEAN_KEYS = frozenset(
|
||||
{
|
||||
"gemini_native_audio",
|
||||
"gemini_audio_only_live",
|
||||
"uses_embed_content",
|
||||
"use_openai_responses_path",
|
||||
"bedrock_converse_supports_strict_tools",
|
||||
}
|
||||
)
|
||||
|
||||
OBJECT_KEYS: dict[str, JsonSchema] = {
|
||||
"search_context_cost_per_query": {
|
||||
"type": "object",
|
||||
"description": "USD cost per web search query, keyed by search context size.",
|
||||
"properties": {
|
||||
"search_context_size_low": NONNEG_NUMBER,
|
||||
"search_context_size_medium": NONNEG_NUMBER,
|
||||
"search_context_size_high": NONNEG_NUMBER,
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"description": "Free-form notes about the entry (e.g. pricing derivation).",
|
||||
},
|
||||
"provider_specific_entry": {
|
||||
"type": "object",
|
||||
"description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
|
||||
},
|
||||
}
|
||||
|
||||
ARRAY_KEYS: dict[str, JsonSchema] = {
|
||||
"supported_endpoints": {
|
||||
"type": "array",
|
||||
"description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
|
||||
"items": STRING,
|
||||
},
|
||||
"supported_modalities": {
|
||||
"type": "array",
|
||||
"description": "Input modalities the model accepts.",
|
||||
"items": {"type": "string", "enum": ["text", "image", "audio", "video"]},
|
||||
},
|
||||
"supported_output_modalities": {
|
||||
"type": "array",
|
||||
"description": "Output modalities the model can produce.",
|
||||
"items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
|
||||
},
|
||||
"supported_regions": {
|
||||
"type": "array",
|
||||
"description": "Cloud regions the model is available in ('global' or region ids).",
|
||||
"items": STRING,
|
||||
},
|
||||
"tiered_pricing": {
|
||||
"type": "array",
|
||||
"description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"range": {
|
||||
"type": "array",
|
||||
"description": "[min, max] prompt-token span this tier applies to.",
|
||||
"items": NONNEG_NUMBER,
|
||||
"minItems": 2,
|
||||
"maxItems": 2,
|
||||
},
|
||||
"max_results_range": {
|
||||
"type": "array",
|
||||
"description": "[min, max] result-count span this tier applies to (search models).",
|
||||
"items": NONNEG_NUMBER,
|
||||
"minItems": 2,
|
||||
"maxItems": 2,
|
||||
},
|
||||
"input_cost_per_token": NONNEG_NUMBER,
|
||||
"output_cost_per_token": NONNEG_NUMBER,
|
||||
"output_cost_per_reasoning_token": NONNEG_NUMBER,
|
||||
"cache_read_input_token_cost": NONNEG_NUMBER,
|
||||
"input_cost_per_query": NONNEG_NUMBER,
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
INTEGER_KEYS: dict[str, JsonSchema] = {
|
||||
"max_tokens": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.",
|
||||
},
|
||||
"max_input_tokens": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Maximum prompt/context tokens the model accepts.",
|
||||
},
|
||||
"max_output_tokens": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Maximum tokens the model can generate in one response.",
|
||||
},
|
||||
"output_vector_size": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Embedding dimension for embedding models.",
|
||||
},
|
||||
"prompt_cache_min_tokens": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
|
||||
},
|
||||
"tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
|
||||
"rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
|
||||
}
|
||||
|
||||
NUMBER_KEYS: dict[str, JsonSchema] = {
|
||||
"regional_processing_uplift_multiplier_eu": {
|
||||
"type": "number",
|
||||
"minimum": 1,
|
||||
"description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).",
|
||||
},
|
||||
"regional_processing_uplift_multiplier_us": {
|
||||
"type": "number",
|
||||
"minimum": 1,
|
||||
"description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
|
||||
},
|
||||
}
|
||||
|
||||
COST_DESCRIPTIONS: dict[str, str] = {
|
||||
"input_cost_per_token": "USD per prompt token.",
|
||||
"output_cost_per_token": "USD per generated token.",
|
||||
"output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
|
||||
"cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
|
||||
"cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
|
||||
"input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
|
||||
"output_cost_per_token_batches": "USD per generated token via the provider's batch API.",
|
||||
}
|
||||
|
||||
|
||||
def cost_description(key: str) -> Optional[str]:
|
||||
if key in COST_DESCRIPTIONS:
|
||||
return COST_DESCRIPTIONS[key]
|
||||
if key.endswith("_flex"):
|
||||
return "Flex service-tier rate for the same-named base field."
|
||||
if key.endswith("_priority"):
|
||||
return "Priority service-tier rate for the same-named base field."
|
||||
if "_above_" in key:
|
||||
return "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
return None
|
||||
|
||||
|
||||
def cost_schema(key: str) -> JsonSchema:
|
||||
description = cost_description(key)
|
||||
return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER)
|
||||
|
||||
|
||||
def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
|
||||
return {
|
||||
"litellm_provider": {
|
||||
"type": "string",
|
||||
"description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",
|
||||
},
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"description": "Primary API surface / task type of the model.",
|
||||
"enum": list(modes),
|
||||
},
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "URL of the provider pricing/model page this entry was taken from.",
|
||||
},
|
||||
"deprecation_date": {
|
||||
"type": "string",
|
||||
"description": "Date the provider deprecates the model, YYYY-MM-DD.",
|
||||
"pattern": "^\\d{4}-\\d{2}-\\d{2}$",
|
||||
},
|
||||
"web_search_billing_unit": {
|
||||
"type": "string",
|
||||
"description": "Whether web search is billed per query or per prompt.",
|
||||
"enum": ["per_query", "per_prompt"],
|
||||
},
|
||||
"bedrock_output_config_effort_ceiling": {
|
||||
"type": "string",
|
||||
"description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
|
||||
"enum": ["low", "medium", "high", "max", "xhigh"],
|
||||
},
|
||||
"comment": STRING,
|
||||
"audio_transcription_config": STRING,
|
||||
}
|
||||
|
||||
|
||||
def classify(key: str, modes: tuple) -> Optional[JsonSchema]:
|
||||
curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS}
|
||||
if key in curated:
|
||||
return curated[key]
|
||||
if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS:
|
||||
return BOOLEAN
|
||||
if "cost" in key:
|
||||
return cost_schema(key)
|
||||
return None
|
||||
|
||||
|
||||
def build_schema(prices: dict) -> JsonSchema:
|
||||
entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS}
|
||||
all_keys = tuple(sorted({key for entry in entries.values() for key in entry}))
|
||||
modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry}))
|
||||
unclassified = tuple(key for key in all_keys if classify(key, modes) is None)
|
||||
if unclassified:
|
||||
raise SystemExit(
|
||||
f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. "
|
||||
f"Add them to the key tables in {Path(__file__).name} and rerun it."
|
||||
)
|
||||
entry_properties = {key: classify(key, modes) for key in all_keys}
|
||||
return {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "LiteLLM model_prices_and_context_window.json",
|
||||
"description": (
|
||||
"Schema for LiteLLM's model price and context window registry "
|
||||
"(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). "
|
||||
"Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, "
|
||||
"optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. "
|
||||
"All costs are USD per unit. New optional fields are added regularly, so consumers should "
|
||||
"ignore unknown fields rather than reject them."
|
||||
),
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sample_spec": {
|
||||
"type": "object",
|
||||
"description": (
|
||||
"Documentation placeholder illustrating the entry shape; not a real model and not "
|
||||
"schema-conformant (several values are prose)."
|
||||
),
|
||||
},
|
||||
"fallback_generalizations": {
|
||||
"type": "object",
|
||||
"description": "Regex rules that generalize unknown model ids to known families; not a model entry.",
|
||||
"properties": {
|
||||
"rules": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": STRING,
|
||||
"pattern": STRING,
|
||||
"description": STRING,
|
||||
},
|
||||
"required": ["name", "pattern"],
|
||||
"additionalProperties": True,
|
||||
},
|
||||
}
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
"additionalProperties": {"$ref": "#/$defs/modelEntry"},
|
||||
"$defs": {
|
||||
"modelEntry": {
|
||||
"type": "object",
|
||||
"description": (
|
||||
"Pricing, limits, and capability flags for one model. Fields other than litellm_provider "
|
||||
"are optional; boolean capability flags are simply omitted when unknown or false."
|
||||
),
|
||||
"required": ["litellm_provider"],
|
||||
"properties": entry_properties,
|
||||
"additionalProperties": True,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render(schema: JsonSchema) -> str:
|
||||
return json.dumps(schema, indent=2) + "\n"
|
||||
|
||||
|
||||
def validation_errors(prices: dict, schema: JsonSchema) -> tuple:
|
||||
validator = jsonschema.Draft202012Validator(schema)
|
||||
return tuple(
|
||||
f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
|
||||
for error in validator.iter_errors(prices)
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
check = "--check" in sys.argv[1:]
|
||||
prices = json.loads(PRICES_PATH.read_text())
|
||||
rendered = render(build_schema(prices))
|
||||
errors = validation_errors(prices, json.loads(rendered))
|
||||
if errors:
|
||||
print(f"{PRICES_PATH.name} does not validate against the generated schema:")
|
||||
print("\n".join(errors[:20]))
|
||||
return 1
|
||||
if not check:
|
||||
SCHEMA_PATH.write_text(rendered)
|
||||
print(f"wrote {SCHEMA_PATH}")
|
||||
return 0
|
||||
if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered:
|
||||
print(
|
||||
f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. "
|
||||
f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result."
|
||||
)
|
||||
return 1
|
||||
print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
741
model_prices_and_context_window.schema.json
Normal file
741
model_prices_and_context_window.schema.json
Normal file
|
|
@ -0,0 +1,741 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "LiteLLM model_prices_and_context_window.json",
|
||||
"description": "Schema for LiteLLM's model price and context window registry (https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. All costs are USD per unit. New optional fields are added regularly, so consumers should ignore unknown fields rather than reject them.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sample_spec": {
|
||||
"type": "object",
|
||||
"description": "Documentation placeholder illustrating the entry shape; not a real model and not schema-conformant (several values are prose)."
|
||||
},
|
||||
"fallback_generalizations": {
|
||||
"type": "object",
|
||||
"description": "Regex rules that generalize unknown model ids to known families; not a model entry.",
|
||||
"properties": {
|
||||
"rules": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"pattern": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"pattern"
|
||||
],
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"additionalProperties": {
|
||||
"$ref": "#/$defs/modelEntry"
|
||||
},
|
||||
"$defs": {
|
||||
"modelEntry": {
|
||||
"type": "object",
|
||||
"description": "Pricing, limits, and capability flags for one model. Fields other than litellm_provider are optional; boolean capability flags are simply omitted when unknown or false.",
|
||||
"required": [
|
||||
"litellm_provider"
|
||||
],
|
||||
"properties": {
|
||||
"annotation_cost_per_page": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"audio_transcription_config": {
|
||||
"type": "string"
|
||||
},
|
||||
"bedrock_converse_supports_strict_tools": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"bedrock_output_config_effort_ceiling": {
|
||||
"type": "string",
|
||||
"description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
|
||||
"enum": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"max",
|
||||
"xhigh"
|
||||
]
|
||||
},
|
||||
"cache_creation_input_audio_token_cost": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"cache_creation_input_token_cost": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "USD per token written to the provider's prompt cache."
|
||||
},
|
||||
"cache_creation_input_token_cost_above_1hr": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"cache_creation_input_token_cost_above_200k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"cache_creation_input_token_cost_above_272k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"cache_creation_input_token_cost_flex": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Flex service-tier rate for the same-named base field."
|
||||
},
|
||||
"cache_creation_input_token_cost_priority": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Priority service-tier rate for the same-named base field."
|
||||
},
|
||||
"cache_read_input_audio_token_cost": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"cache_read_input_token_cost": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "USD per prompt token served from the provider's prompt cache."
|
||||
},
|
||||
"cache_read_input_token_cost_above_200k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Priority service-tier rate for the same-named base field."
|
||||
},
|
||||
"cache_read_input_token_cost_above_272k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Priority service-tier rate for the same-named base field."
|
||||
},
|
||||
"cache_read_input_token_cost_above_512k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"cache_read_input_token_cost_flex": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Flex service-tier rate for the same-named base field."
|
||||
},
|
||||
"cache_read_input_token_cost_priority": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Priority service-tier rate for the same-named base field."
|
||||
},
|
||||
"citation_cost_per_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"code_interpreter_cost_per_session": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"comment": {
|
||||
"type": "string"
|
||||
},
|
||||
"deprecation_date": {
|
||||
"type": "string",
|
||||
"description": "Date the provider deprecates the model, YYYY-MM-DD.",
|
||||
"pattern": "^\\d{4}-\\d{2}-\\d{2}$"
|
||||
},
|
||||
"gemini_audio_only_live": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"gemini_native_audio": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"input_cost_per_audio_per_second": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_audio_per_second_above_128k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_audio_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_audio_token_priority": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Priority service-tier rate for the same-named base field."
|
||||
},
|
||||
"input_cost_per_character": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_character_above_128k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_image": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_image_above_128k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_image_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_pixel": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_query": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_request": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_second": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_token": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "USD per prompt token."
|
||||
},
|
||||
"input_cost_per_token_above_128k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_token_above_200k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_token_above_200k_tokens_priority": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Priority service-tier rate for the same-named base field."
|
||||
},
|
||||
"input_cost_per_token_above_256k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_token_above_272k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_token_above_272k_tokens_priority": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Priority service-tier rate for the same-named base field."
|
||||
},
|
||||
"input_cost_per_token_above_512k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_token_batches": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "USD per prompt token via the provider's batch API."
|
||||
},
|
||||
"input_cost_per_token_cache_hit": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_token_flex": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Flex service-tier rate for the same-named base field."
|
||||
},
|
||||
"input_cost_per_token_priority": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Priority service-tier rate for the same-named base field."
|
||||
},
|
||||
"input_cost_per_video_per_second": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_video_per_second_above_128k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_video_per_second_above_15s_interval": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_video_per_second_above_8s_interval": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_dbu_cost_per_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"litellm_provider": {
|
||||
"type": "string",
|
||||
"description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers."
|
||||
},
|
||||
"max_input_tokens": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Maximum prompt/context tokens the model accepts."
|
||||
},
|
||||
"max_output_tokens": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Maximum tokens the model can generate in one response."
|
||||
},
|
||||
"max_tokens": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Legacy field: max output tokens if the provider specifies it, else max input tokens."
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"description": "Free-form notes about the entry (e.g. pricing derivation)."
|
||||
},
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"description": "Primary API surface / task type of the model.",
|
||||
"enum": [
|
||||
"audio_speech",
|
||||
"audio_transcription",
|
||||
"chat",
|
||||
"completion",
|
||||
"embedding",
|
||||
"image_edit",
|
||||
"image_generation",
|
||||
"moderation",
|
||||
"ocr",
|
||||
"realtime",
|
||||
"rerank",
|
||||
"responses",
|
||||
"search",
|
||||
"vector_store",
|
||||
"video_generation"
|
||||
]
|
||||
},
|
||||
"ocr_cost_per_credit": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"ocr_cost_per_page": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_cost_per_audio_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_cost_per_character": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_cost_per_character_above_128k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"output_cost_per_image": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_cost_per_image_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_cost_per_pixel": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_cost_per_reasoning_token": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "USD per reasoning/thinking token, when billed separately."
|
||||
},
|
||||
"output_cost_per_second": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_cost_per_second_1080p": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_cost_per_token": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "USD per generated token."
|
||||
},
|
||||
"output_cost_per_token_above_128k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"output_cost_per_token_above_200k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"output_cost_per_token_above_200k_tokens_priority": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Priority service-tier rate for the same-named base field."
|
||||
},
|
||||
"output_cost_per_token_above_256k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"output_cost_per_token_above_272k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"output_cost_per_token_above_272k_tokens_priority": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Priority service-tier rate for the same-named base field."
|
||||
},
|
||||
"output_cost_per_token_above_512k_tokens": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"output_cost_per_token_batches": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "USD per generated token via the provider's batch API."
|
||||
},
|
||||
"output_cost_per_token_flex": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Flex service-tier rate for the same-named base field."
|
||||
},
|
||||
"output_cost_per_token_priority": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Priority service-tier rate for the same-named base field."
|
||||
},
|
||||
"output_cost_per_video_per_second": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_cost_per_video_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_dbu_cost_per_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_vector_size": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Embedding dimension for embedding models."
|
||||
},
|
||||
"prompt_cache_min_tokens": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Smallest prefix the provider will actually cache; absent means the provider default applies."
|
||||
},
|
||||
"provider_specific_entry": {
|
||||
"type": "object",
|
||||
"description": "Provider-internal routing hints (e.g. bedrock_invocation_schema)."
|
||||
},
|
||||
"regional_processing_uplift_multiplier_eu": {
|
||||
"type": "number",
|
||||
"minimum": 1,
|
||||
"description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%)."
|
||||
},
|
||||
"regional_processing_uplift_multiplier_us": {
|
||||
"type": "number",
|
||||
"minimum": 1,
|
||||
"description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%)."
|
||||
},
|
||||
"rpm": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Provider default requests-per-minute limit."
|
||||
},
|
||||
"search_context_cost_per_query": {
|
||||
"type": "object",
|
||||
"description": "USD cost per web search query, keyed by search context size.",
|
||||
"properties": {
|
||||
"search_context_size_low": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"search_context_size_medium": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"search_context_size_high": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "URL of the provider pricing/model page this entry was taken from."
|
||||
},
|
||||
"supported_endpoints": {
|
||||
"type": "array",
|
||||
"description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"supported_modalities": {
|
||||
"type": "array",
|
||||
"description": "Input modalities the model accepts.",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"text",
|
||||
"image",
|
||||
"audio",
|
||||
"video"
|
||||
]
|
||||
}
|
||||
},
|
||||
"supported_output_modalities": {
|
||||
"type": "array",
|
||||
"description": "Output modalities the model can produce.",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"text",
|
||||
"image",
|
||||
"audio",
|
||||
"video",
|
||||
"code"
|
||||
]
|
||||
}
|
||||
},
|
||||
"supported_regions": {
|
||||
"type": "array",
|
||||
"description": "Cloud regions the model is available in ('global' or region ids).",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"supports_adaptive_thinking": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_assistant_prefill": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_audio_input": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_audio_output": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_computer_use": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_embedding_image_input": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_function_calling": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_image_input": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_image_size": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_low_reasoning_effort": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_max_reasoning_effort": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_mid_conversation_system": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_minimal_reasoning_effort": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_multimodal": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_native_streaming": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_native_structured_output": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_none_reasoning_effort": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_nova_canvas_image_edit": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_output_config": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_parallel_function_calling": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_parallel_tool_use_config": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_pdf_input": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_prompt_caching": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_reasoning": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_response_schema": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_sampling_params": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_speed": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_system_messages": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_tool_choice": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_url_context": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_video_input": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_vision": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_web_search": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"supports_xhigh_reasoning_effort": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"tiered_pricing": {
|
||||
"type": "array",
|
||||
"description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"range": {
|
||||
"type": "array",
|
||||
"description": "[min, max] prompt-token span this tier applies to.",
|
||||
"items": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"minItems": 2,
|
||||
"maxItems": 2
|
||||
},
|
||||
"max_results_range": {
|
||||
"type": "array",
|
||||
"description": "[min, max] result-count span this tier applies to (search models).",
|
||||
"items": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"minItems": 2,
|
||||
"maxItems": 2
|
||||
},
|
||||
"input_cost_per_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_cost_per_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_cost_per_reasoning_token": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"cache_read_input_token_cost": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_query": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"tpm": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Provider default tokens-per-minute limit."
|
||||
},
|
||||
"use_openai_responses_path": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"uses_embed_content": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"web_search_billing_unit": {
|
||||
"type": "string",
|
||||
"description": "Whether web search is billed per query or per prompt.",
|
||||
"enum": [
|
||||
"per_query",
|
||||
"per_prompt"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
85
tests/test_litellm/test_model_prices_schema.py
Normal file
85
tests/test_litellm/test_model_prices_schema.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import jsonschema
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py"
|
||||
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
|
||||
|
||||
|
||||
def load_generator():
|
||||
spec = importlib.util.spec_from_file_location("generate_model_prices_schema", GENERATOR_PATH)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def committed_schema() -> dict:
|
||||
return json.loads(SCHEMA_PATH.read_text())
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def prices() -> dict:
|
||||
return json.loads(PRICES_PATH.read_text())
|
||||
|
||||
|
||||
def test_committed_schema_matches_generator_output(prices: dict, committed_schema: dict):
|
||||
generator = load_generator()
|
||||
regenerated = json.loads(generator.render(generator.build_schema(prices)))
|
||||
assert regenerated == committed_schema, (
|
||||
"model_prices_and_context_window.schema.json is out of sync; "
|
||||
"run `python ci_cd/generate_model_prices_schema.py` and commit the result"
|
||||
)
|
||||
|
||||
|
||||
def test_prices_file_validates_against_committed_schema(prices: dict, committed_schema: dict):
|
||||
validator = jsonschema.Draft202012Validator(committed_schema)
|
||||
errors = [
|
||||
f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
|
||||
for error in validator.iter_errors(prices)
|
||||
]
|
||||
assert errors == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"entry",
|
||||
[
|
||||
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": "0.01"},
|
||||
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": -1},
|
||||
{"litellm_provider": "openai", "mode": "not_a_real_mode"},
|
||||
{"mode": "chat"},
|
||||
{"litellm_provider": "openai", "deprecation_date": "June 2026"},
|
||||
{"litellm_provider": "openai", "supported_modalities": ["smell"]},
|
||||
{"litellm_provider": "openai", "supports_vision": "yes"},
|
||||
{"litellm_provider": "openai", "max_tokens": 8191.5},
|
||||
{"litellm_provider": "openai", "tiered_pricing": [{"unknown_tier_field": 1}]},
|
||||
],
|
||||
ids=[
|
||||
"cost_as_string",
|
||||
"negative_cost",
|
||||
"unknown_mode",
|
||||
"missing_provider",
|
||||
"non_iso_deprecation_date",
|
||||
"unknown_modality",
|
||||
"boolean_flag_as_string",
|
||||
"fractional_max_tokens",
|
||||
"unknown_tiered_pricing_field",
|
||||
],
|
||||
)
|
||||
def test_schema_rejects_malformed_entries(committed_schema: dict, entry: dict):
|
||||
validator = jsonschema.Draft202012Validator(committed_schema)
|
||||
assert not validator.is_valid({"some-model": entry})
|
||||
|
||||
|
||||
def test_schema_accepts_minimal_and_unknown_optional_fields(committed_schema: dict):
|
||||
validator = jsonschema.Draft202012Validator(committed_schema)
|
||||
assert validator.is_valid({"some-model": {"litellm_provider": "openai"}})
|
||||
assert validator.is_valid({"some-model": {"litellm_provider": "openai", "brand_new_field": {"nested": True}}})
|
||||
Loading…
Add table
Reference in a new issue