mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
ci(cost map): remove the generated model prices JSON schema (#47064)
Backport of #47064 to stable/1.104.x.
Cherry-picked from ae6f7f6613 (main).
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Mateo Wang
mateo-berri
parent
a4be1c1d13
commit
da5a9ffd2e
7 changed files with 27 additions and 705 deletions
|
|
@ -24,12 +24,9 @@ jobs:
|
|||
- name: Update JSON Data
|
||||
run: |
|
||||
uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
|
||||
- name: Regenerate JSON Schema
|
||||
run: |
|
||||
uv run --frozen python ci_cd/generate_model_prices_schema.py
|
||||
- name: Create Pull Request
|
||||
run: |
|
||||
git add model_prices_and_context_window.json model_prices_and_context_window.schema.json
|
||||
git add model_prices_and_context_window.json
|
||||
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
|
||||
gh pr create --title "Update model_prices_and_context_window.json file" \
|
||||
--body "Automated update for model_prices_and_context_window.json" \
|
||||
|
|
|
|||
|
|
@ -41,10 +41,6 @@ jobs:
|
|||
uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
|
||||
env:
|
||||
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
|
||||
- name: Regenerate the JSON schema
|
||||
if: steps.existing.outputs.open_pr == ''
|
||||
run: |
|
||||
uv run --frozen python ci_cd/generate_model_prices_schema.py
|
||||
- name: Create a pull request when the registry changed
|
||||
if: steps.existing.outputs.open_pr == ''
|
||||
run: |
|
||||
|
|
@ -57,8 +53,7 @@ jobs:
|
|||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -b "$branch"
|
||||
git add model_prices_and_context_window.json \
|
||||
litellm/model_prices_and_context_window_backup.json \
|
||||
model_prices_and_context_window.schema.json
|
||||
litellm/model_prices_and_context_window_backup.json
|
||||
git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
|
||||
gh auth setup-git
|
||||
git push origin "$branch"
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
"""Guard the cost map on pull requests.
|
||||
|
||||
Every pull request whose diff against its merge base touches one of the three cost map files gets the file
|
||||
checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
|
||||
map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
|
||||
and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
|
||||
(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
|
||||
files and may only add or update models.
|
||||
Every pull request whose diff against its merge base touches one of the two cost map files gets the file
|
||||
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
|
||||
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
|
||||
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
|
||||
get the file checks and additionally may only touch those two files or the legacy schema
|
||||
path and may only add or update models.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -19,13 +19,12 @@ from dataclasses import dataclass
|
|||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
|
||||
|
||||
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
|
||||
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
|
||||
SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
|
||||
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
|
||||
LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
|
||||
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
|
||||
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
|
||||
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
|
||||
EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
CostMap = dict[str, object]
|
||||
|
|
@ -35,7 +34,6 @@ CostMap = dict[str, object]
|
|||
class Snapshot:
|
||||
cost_map: str
|
||||
backup: str
|
||||
schema: str
|
||||
|
||||
|
||||
def _parse_object(text: str, path: str) -> CostMap | str:
|
||||
|
|
@ -46,13 +44,6 @@ def _parse_object(text: str, path: str) -> CostMap | str:
|
|||
return parsed if isinstance(parsed, dict) else f"{path} must be a JSON object at the root"
|
||||
|
||||
|
||||
def _rendered_schema(cost_map: CostMap) -> str:
|
||||
try:
|
||||
return render(build_schema(cost_map))
|
||||
except SystemExit as error:
|
||||
return str(error)
|
||||
|
||||
|
||||
def _evaluation_output_price_failure(
|
||||
key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
|
||||
) -> str | None:
|
||||
|
|
@ -78,29 +69,13 @@ def _file_failures(
|
|||
*,
|
||||
evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
|
||||
) -> tuple[str, ...]:
|
||||
schema_text: Final = _rendered_schema(head_map)
|
||||
if not schema_text.startswith("{"):
|
||||
return (schema_text,)
|
||||
backup_failure: Final = (
|
||||
()
|
||||
if head.backup == head.cost_map
|
||||
else (f"{BACKUP_PATH} differs from {COST_MAP_PATH}; copy the root file over it",)
|
||||
)
|
||||
schema_failure: Final = (
|
||||
()
|
||||
if head.schema == schema_text
|
||||
else (
|
||||
f"{SCHEMA_PATH} is out of sync with {COST_MAP_PATH}; "
|
||||
"run `python ci_cd/generate_model_prices_schema.py` and commit the result",
|
||||
)
|
||||
)
|
||||
return (
|
||||
*backup_failure,
|
||||
*schema_failure,
|
||||
*(
|
||||
f"{COST_MAP_PATH} does not validate against its schema: {error}"
|
||||
for error in validation_errors(head_map, json.loads(schema_text))[:20]
|
||||
),
|
||||
*(
|
||||
failure
|
||||
for key, entry in head_map.items()
|
||||
|
|
@ -135,7 +110,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
|
|||
*(
|
||||
f"bot PRs may only change the cost map files, not {path}"
|
||||
for path in changed_files
|
||||
if path not in GUARDED_PATHS
|
||||
if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
|
||||
),
|
||||
*(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
|
||||
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),
|
||||
|
|
|
|||
|
|
@ -1,424 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final, Optional
|
||||
|
||||
import jsonschema
|
||||
|
||||
REPO_ROOT = Path(__file__).parent.parent
|
||||
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
|
||||
|
||||
SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"})
|
||||
|
||||
JsonSchema = dict
|
||||
|
||||
NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
|
||||
NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
|
||||
BOOLEAN: JsonSchema = {"type": "boolean"}
|
||||
STRING: JsonSchema = {"type": "string"}
|
||||
NON_BLANK_STRING: Final[JsonSchema] = {"type": "string", "pattern": r"\S"}
|
||||
TIME_WINDOW: Final[JsonSchema] = {"type": "string", "pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"}
|
||||
WEEKDAY_PATTERN: Final = (
|
||||
r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$"
|
||||
)
|
||||
|
||||
EXTRA_BOOLEAN_KEYS = frozenset(
|
||||
{
|
||||
"gemini_native_audio",
|
||||
"gemini_audio_only_live",
|
||||
"uses_embed_content",
|
||||
"use_openai_responses_path",
|
||||
"bedrock_converse_supports_strict_tools",
|
||||
"thinking_always_on",
|
||||
}
|
||||
)
|
||||
|
||||
HOURS_UTC: Final[JsonSchema] = {
|
||||
"description": 'UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight.',
|
||||
"oneOf": [TIME_WINDOW, {"type": "array", "items": TIME_WINDOW, "minItems": 1}],
|
||||
}
|
||||
|
||||
OFF_PEAK_WINDOW: Final[JsonSchema] = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"hours_utc": HOURS_UTC,
|
||||
"weekdays": {
|
||||
"type": "array",
|
||||
"description": "ISO-8601 weekday numbers (1 = Monday .. 7 = Sunday) or English day names the window applies on.",
|
||||
"items": {
|
||||
"oneOf": [
|
||||
{"type": "integer", "minimum": 1, "maximum": 7},
|
||||
{"type": "string", "pattern": WEEKDAY_PATTERN},
|
||||
]
|
||||
},
|
||||
"minItems": 1,
|
||||
},
|
||||
},
|
||||
"required": ["hours_utc"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
||||
OBJECT_KEYS: dict[str, JsonSchema] = {
|
||||
"off_peak_pricing": {
|
||||
"type": "object",
|
||||
"description": "Rates that replace the same-named base fields while the request falls inside the stated UTC windows.",
|
||||
"properties": {
|
||||
"hours_utc": HOURS_UTC,
|
||||
"windows": {"type": "array", "items": OFF_PEAK_WINDOW, "minItems": 1},
|
||||
"weekday_timezone": {
|
||||
"type": "string",
|
||||
"description": "IANA zone the weekdays of each window are read on; defaults to UTC.",
|
||||
},
|
||||
"input_cost_per_token": NONNEG_NUMBER,
|
||||
"output_cost_per_token": NONNEG_NUMBER,
|
||||
"output_cost_per_reasoning_token": NONNEG_NUMBER,
|
||||
"cache_read_input_token_cost": NONNEG_NUMBER,
|
||||
"cache_creation_input_token_cost": NONNEG_NUMBER,
|
||||
},
|
||||
"anyOf": [{"required": ["hours_utc"]}, {"required": ["windows"]}],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"search_context_cost_per_query": {
|
||||
"type": "object",
|
||||
"description": "USD cost per web search query, keyed by search context size.",
|
||||
"properties": {
|
||||
"search_context_size_low": NONNEG_NUMBER,
|
||||
"search_context_size_medium": NONNEG_NUMBER,
|
||||
"search_context_size_high": NONNEG_NUMBER,
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"guardrail_cost_per_unit": {
|
||||
"type": "object",
|
||||
"description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).",
|
||||
"additionalProperties": NONNEG_NUMBER,
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"description": "Free-form notes about the entry (e.g. pricing derivation).",
|
||||
},
|
||||
"provider_specific_entry": {
|
||||
"type": "object",
|
||||
"description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
|
||||
},
|
||||
"token_count_model": {
|
||||
"type": "object",
|
||||
"required": ["model"],
|
||||
"additionalProperties": False,
|
||||
"properties": {
|
||||
"model": NON_BLANK_STRING,
|
||||
"region": NON_BLANK_STRING,
|
||||
},
|
||||
"description": "Model that counts tokens for this one, and the AWS region its count API runs in.",
|
||||
},
|
||||
}
|
||||
|
||||
ARRAY_KEYS: dict[str, JsonSchema] = {
|
||||
"supported_audio_formats": {
|
||||
"type": "array",
|
||||
"description": "Audio container formats the model can return.",
|
||||
"items": {"type": "string", "enum": ["mp3", "wav"]},
|
||||
},
|
||||
"supported_endpoints": {
|
||||
"type": "array",
|
||||
"description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
|
||||
"items": STRING,
|
||||
},
|
||||
"supported_modalities": {
|
||||
"type": "array",
|
||||
"description": "Input modalities the model accepts.",
|
||||
"items": {"type": "string", "enum": ["text", "image", "audio", "video"]},
|
||||
},
|
||||
"supported_output_modalities": {
|
||||
"type": "array",
|
||||
"description": "Output modalities the model can produce.",
|
||||
"items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
|
||||
},
|
||||
"reasoning_effort_levels": {
|
||||
"type": "array",
|
||||
"description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
|
||||
"items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
|
||||
},
|
||||
"supported_regions": {
|
||||
"type": "array",
|
||||
"description": "Cloud regions the model is available in ('global' or region ids).",
|
||||
"items": STRING,
|
||||
},
|
||||
"tiered_pricing": {
|
||||
"type": "array",
|
||||
"description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"range": {
|
||||
"type": "array",
|
||||
"description": "[min, max] prompt-token span this tier applies to.",
|
||||
"items": NONNEG_NUMBER,
|
||||
"minItems": 2,
|
||||
"maxItems": 2,
|
||||
},
|
||||
"max_results_range": {
|
||||
"type": "array",
|
||||
"description": "[min, max] result-count span this tier applies to (search models).",
|
||||
"items": NONNEG_NUMBER,
|
||||
"minItems": 2,
|
||||
"maxItems": 2,
|
||||
},
|
||||
"input_cost_per_token": NONNEG_NUMBER,
|
||||
"output_cost_per_token": NONNEG_NUMBER,
|
||||
"output_cost_per_reasoning_token": NONNEG_NUMBER,
|
||||
"cache_read_input_token_cost": NONNEG_NUMBER,
|
||||
"cache_creation_input_token_cost": NONNEG_NUMBER,
|
||||
"input_cost_per_query": NONNEG_NUMBER,
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
INTEGER_KEYS: dict[str, JsonSchema] = {
|
||||
"max_tokens": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.",
|
||||
},
|
||||
"max_input_tokens": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Maximum prompt/context tokens the model accepts.",
|
||||
},
|
||||
"max_output_tokens": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Maximum tokens the model can generate in one response.",
|
||||
},
|
||||
"output_vector_size": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Embedding dimension for embedding models.",
|
||||
},
|
||||
"prompt_cache_min_tokens": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
|
||||
},
|
||||
"token_count_framing": {
|
||||
**NONNEG_INTEGER,
|
||||
"description": "Tokens added once per request to a local count estimate, for the provider's chat framing.",
|
||||
},
|
||||
"tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
|
||||
"rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
|
||||
}
|
||||
|
||||
NUMBER_KEYS: dict[str, JsonSchema] = {
|
||||
"regional_processing_uplift_multiplier_eu": {
|
||||
"type": "number",
|
||||
"minimum": 1,
|
||||
"description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).",
|
||||
},
|
||||
"regional_processing_uplift_multiplier_us": {
|
||||
"type": "number",
|
||||
"minimum": 1,
|
||||
"description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
|
||||
},
|
||||
"regional_endpoint_uplift_multiplier": {
|
||||
"type": "number",
|
||||
"minimum": 1,
|
||||
"description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).",
|
||||
},
|
||||
}
|
||||
|
||||
COST_DESCRIPTIONS: dict[str, str] = {
|
||||
"input_cost_per_token": "USD per prompt token.",
|
||||
"output_cost_per_token": "USD per generated token.",
|
||||
"output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
|
||||
"google_maps_grounding_cost_per_query": (
|
||||
"USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
|
||||
),
|
||||
"cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
|
||||
"cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
|
||||
"input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
|
||||
"output_cost_per_token_batches": "USD per generated token via the provider's batch API.",
|
||||
}
|
||||
|
||||
|
||||
def cost_description(key: str) -> Optional[str]:
|
||||
if key in COST_DESCRIPTIONS:
|
||||
return COST_DESCRIPTIONS[key]
|
||||
if key.endswith("_flex"):
|
||||
return "Flex service-tier rate for the same-named base field."
|
||||
if key.endswith("_priority"):
|
||||
return "Priority service-tier rate for the same-named base field."
|
||||
if "_above_" in key:
|
||||
return "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
return None
|
||||
|
||||
|
||||
def cost_schema(key: str) -> JsonSchema:
|
||||
description = cost_description(key)
|
||||
return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER)
|
||||
|
||||
|
||||
def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
|
||||
return {
|
||||
"litellm_provider": {
|
||||
"type": "string",
|
||||
"description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",
|
||||
},
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"description": "Primary API surface / task type of the model.",
|
||||
"enum": list(modes),
|
||||
},
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "URL of the provider pricing/model page this entry was taken from.",
|
||||
},
|
||||
"deprecation_date": {
|
||||
"type": "string",
|
||||
"description": "Date the provider deprecates the model, YYYY-MM-DD.",
|
||||
"format": "date",
|
||||
"pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$",
|
||||
},
|
||||
"web_search_billing_unit": {
|
||||
"type": "string",
|
||||
"description": "Whether web search is billed per query or per prompt.",
|
||||
"enum": ["per_query", "per_prompt"],
|
||||
},
|
||||
"bedrock_output_config_effort_ceiling": {
|
||||
"type": "string",
|
||||
"description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
|
||||
"enum": ["low", "medium", "high", "max", "xhigh"],
|
||||
},
|
||||
"default_reasoning_effort": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Reasoning effort the provider applies when the request omits reasoning_effort. "
|
||||
"Gates whether a non-default temperature or the top_p/logprobs sampling params are "
|
||||
"accepted, which hold only when the effort resolves to 'none'."
|
||||
),
|
||||
"enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
|
||||
},
|
||||
"comment": STRING,
|
||||
"audio_transcription_config": STRING,
|
||||
"vertex_ai_audio_api": {
|
||||
"type": "string",
|
||||
"enum": ["lyria_predict", "lyria_interactions"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def classify(key: str, modes: tuple) -> Optional[JsonSchema]:
|
||||
curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS}
|
||||
if key in curated:
|
||||
return curated[key]
|
||||
if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS:
|
||||
return BOOLEAN
|
||||
if "cost" in key:
|
||||
return cost_schema(key)
|
||||
return None
|
||||
|
||||
|
||||
def build_schema(prices: dict) -> JsonSchema:
|
||||
entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS}
|
||||
all_keys = tuple(sorted({key for entry in entries.values() for key in entry}))
|
||||
modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry}))
|
||||
unclassified = tuple(key for key in all_keys if classify(key, modes) is None)
|
||||
if unclassified:
|
||||
raise SystemExit(
|
||||
f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. "
|
||||
f"Add them to the key tables in {Path(__file__).name} and rerun it."
|
||||
)
|
||||
entry_properties = {key: classify(key, modes) for key in all_keys}
|
||||
return {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "LiteLLM model_prices_and_context_window.json",
|
||||
"description": (
|
||||
"Schema for LiteLLM's model price and context window registry "
|
||||
"(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). "
|
||||
"Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, "
|
||||
"optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. "
|
||||
"All costs are USD per unit. New optional fields are added regularly, so consumers should "
|
||||
"ignore unknown fields rather than reject them."
|
||||
),
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sample_spec": {
|
||||
"type": "object",
|
||||
"description": (
|
||||
"Documentation placeholder illustrating the entry shape; not a real model and not "
|
||||
"schema-conformant (several values are prose)."
|
||||
),
|
||||
},
|
||||
"fallback_generalizations": {
|
||||
"type": "object",
|
||||
"description": "Regex rules that generalize unknown model ids to known families; not a model entry.",
|
||||
"properties": {
|
||||
"rules": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": STRING,
|
||||
"pattern": STRING,
|
||||
"description": STRING,
|
||||
},
|
||||
"required": ["name", "pattern"],
|
||||
"additionalProperties": True,
|
||||
},
|
||||
}
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
"additionalProperties": {"$ref": "#/$defs/modelEntry"},
|
||||
"$defs": {
|
||||
"modelEntry": {
|
||||
"type": "object",
|
||||
"description": (
|
||||
"Pricing, limits, and capability flags for one model. Fields other than litellm_provider "
|
||||
"are optional; boolean capability flags are simply omitted when unknown or false."
|
||||
),
|
||||
"required": ["litellm_provider"],
|
||||
"properties": entry_properties,
|
||||
"additionalProperties": True,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render(schema: JsonSchema) -> str:
|
||||
return json.dumps(schema, indent=2) + "\n"
|
||||
|
||||
|
||||
def validation_errors(prices: dict, schema: JsonSchema) -> tuple:
|
||||
validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER)
|
||||
return tuple(
|
||||
f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
|
||||
for error in validator.iter_errors(prices)
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
check = "--check" in sys.argv[1:]
|
||||
prices = json.loads(PRICES_PATH.read_text())
|
||||
rendered = render(build_schema(prices))
|
||||
errors = validation_errors(prices, json.loads(rendered))
|
||||
if errors:
|
||||
print(f"{PRICES_PATH.name} does not validate against the generated schema:")
|
||||
print("\n".join(errors[:20]))
|
||||
return 1
|
||||
if not check:
|
||||
SCHEMA_PATH.write_text(rendered)
|
||||
print(f"wrote {SCHEMA_PATH}")
|
||||
return 0
|
||||
if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered:
|
||||
print(
|
||||
f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. "
|
||||
f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result."
|
||||
)
|
||||
return 1
|
||||
print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -2,5 +2,4 @@
|
|||
|
||||
For `model_prices_and_context_window.json` validation, we should eventually:
|
||||
|
||||
- Remove any schema file like `model_prices_and_context_window.schema.json`
|
||||
- Stop skipping this crate's tests
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ def _load(name: str) -> ModuleType:
|
|||
return module
|
||||
|
||||
|
||||
schema_module: Final = _load("generate_model_prices_schema")
|
||||
guard: Final = _load("cost_map_guard")
|
||||
|
||||
MAP_FILES: Final = (guard.COST_MAP_PATH,)
|
||||
|
|
@ -52,12 +51,9 @@ def _serialize(cost_map: dict[str, object]) -> str:
|
|||
return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def _snapshot(cost_map: dict[str, object], backup: str | None = None, schema: str | None = None) -> object:
|
||||
def _snapshot(cost_map: dict[str, object], backup: str | None = None) -> object:
|
||||
text = _serialize(cost_map)
|
||||
rendered = schema_module.render(schema_module.build_schema(cost_map))
|
||||
return guard.Snapshot(
|
||||
cost_map=text, backup=text if backup is None else backup, schema=rendered if schema is None else schema
|
||||
)
|
||||
return guard.Snapshot(cost_map=text, backup=text if backup is None else backup)
|
||||
|
||||
|
||||
BASE: Final = _snapshot(BASE_MAP)
|
||||
|
|
@ -135,14 +131,14 @@ def test_bot_may_add_and_reprice_models() -> None:
|
|||
|
||||
|
||||
def test_broken_json_is_reported() -> None:
|
||||
head = guard.Snapshot(cost_map="{not json", backup="{not json", schema="{}")
|
||||
head = guard.Snapshot(cost_map="{not json", backup="{not json")
|
||||
assert _failures(head, bot=False) == (
|
||||
f"{guard.COST_MAP_PATH} is not valid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)",
|
||||
)
|
||||
|
||||
|
||||
def test_non_object_root_is_reported() -> None:
|
||||
head = guard.Snapshot(cost_map="[]", backup="[]", schema="{}")
|
||||
head = guard.Snapshot(cost_map="[]", backup="[]")
|
||||
assert _failures(head, bot=False) == (f"{guard.COST_MAP_PATH} must be a JSON object at the root",)
|
||||
|
||||
|
||||
|
|
@ -151,28 +147,7 @@ def test_backup_drift_is_reported() -> None:
|
|||
assert [failure for failure in _failures(head, bot=False) if failure.startswith(guard.BACKUP_PATH)]
|
||||
|
||||
|
||||
def test_schema_out_of_sync_is_reported() -> None:
|
||||
head = _snapshot({**BASE_MAP, "openrouter/c": _entry(supports_audio_input=True)}, schema=BASE.schema)
|
||||
assert [failure for failure in _failures(head, bot=False) if failure.startswith(guard.SCHEMA_PATH)]
|
||||
|
||||
|
||||
def test_schema_validation_errors_are_reported() -> None:
|
||||
head = _snapshot({**BASE_MAP, "openrouter/c": _entry(-1e-06)})
|
||||
prefix = f"{guard.COST_MAP_PATH} does not validate against its schema: openrouter/c."
|
||||
assert [failure.removeprefix(prefix).split(":")[0] for failure in _failures(head, bot=False)] == [
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
]
|
||||
|
||||
|
||||
def test_unclassified_entry_key_is_reported() -> None:
|
||||
text = _serialize({**BASE_MAP, "openrouter/c": _entry(weird_thing=1)})
|
||||
head = guard.Snapshot(cost_map=text, backup=text, schema=BASE.schema)
|
||||
(failure,) = _failures(head, bot=False)
|
||||
assert "Unclassified keys" in failure and "weird_thing" in failure
|
||||
|
||||
|
||||
STALE_HEAD: Final = _snapshot(BASE_MAP, backup=_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)}), schema="{}")
|
||||
STALE_HEAD: Final = _snapshot(BASE_MAP, backup=_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)}))
|
||||
CODE_ONLY: Final = (
|
||||
"litellm/utils.py",
|
||||
"tests/test_litellm/test_utils.py",
|
||||
|
|
@ -181,7 +156,7 @@ CODE_ONLY: Final = (
|
|||
|
||||
|
||||
def test_human_pr_that_leaves_the_cost_map_alone_skips_the_file_checks() -> None:
|
||||
unparseable: Final = guard.Snapshot(cost_map="{not json", backup="", schema="")
|
||||
unparseable: Final = guard.Snapshot(cost_map="{not json", backup="")
|
||||
assert _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=False) == ()
|
||||
assert _failures(STALE_HEAD, changed_files=(), bot=False) == ()
|
||||
assert _failures(unparseable, changed_files=CODE_ONLY, bot=False) == ()
|
||||
|
|
@ -191,18 +166,16 @@ def test_human_pr_that_leaves_the_cost_map_alone_skips_the_file_checks() -> None
|
|||
def test_touching_any_cost_map_file_keeps_the_file_checks(guarded_path: str) -> None:
|
||||
failures: Final = _failures(STALE_HEAD, changed_files=(*CODE_ONLY, guarded_path), bot=False)
|
||||
assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)]
|
||||
assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)]
|
||||
|
||||
|
||||
def test_bot_pr_always_gets_the_file_checks() -> None:
|
||||
failures: Final = _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=True)
|
||||
assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)]
|
||||
assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)]
|
||||
|
||||
|
||||
def test_contract_names_the_skip() -> None:
|
||||
assert guard.contract_for(False, CODE_ONLY) == "human PR, cost map untouched"
|
||||
assert guard.contract_for(False, (*CODE_ONLY, guard.SCHEMA_PATH)) == "human PR, file checks only"
|
||||
assert guard.contract_for(False, (*CODE_ONLY, guard.COST_MAP_PATH)) == "human PR, file checks only"
|
||||
assert guard.contract_for(True, CODE_ONLY) == "bot contract enforced"
|
||||
|
||||
|
||||
|
|
@ -215,6 +188,13 @@ def test_bot_may_only_touch_the_cost_map_files() -> None:
|
|||
)
|
||||
|
||||
|
||||
def test_bot_may_update_cost_map_and_legacy_schema() -> None:
|
||||
head: Final = _snapshot({**BASE_MAP, "openrouter/c": _entry()})
|
||||
changed: Final = (*guard.GUARDED_PATHS, guard.LEGACY_SCHEMA_PATH)
|
||||
|
||||
assert _failures(head, changed_files=changed) == ()
|
||||
|
||||
|
||||
def test_bot_may_not_remove_models() -> None:
|
||||
head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"})
|
||||
assert _failures(head, bot=False) == ()
|
||||
|
|
@ -238,7 +218,6 @@ def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str:
|
|||
(repo / guard.COST_MAP_PATH).write_text(text)
|
||||
(repo / guard.BACKUP_PATH).parent.mkdir(exist_ok=True)
|
||||
(repo / guard.BACKUP_PATH).write_text(text)
|
||||
(repo / guard.SCHEMA_PATH).write_text(schema_module.render(schema_module.build_schema(cost_map)))
|
||||
return _git_commit(repo, message)
|
||||
|
||||
|
||||
|
|
@ -292,8 +271,7 @@ def test_main_skips_the_file_checks_on_a_stale_base_the_pr_never_touched(tmp_pat
|
|||
subprocess.run(("git", "init", "-q", str(tmp_path)), check=True)
|
||||
_commit(tmp_path, BASE_MAP, "base")
|
||||
(tmp_path / guard.BACKUP_PATH).write_text(_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)}))
|
||||
(tmp_path / guard.SCHEMA_PATH).write_text("{}")
|
||||
stale_base: Final = _commit_code_only(tmp_path, "stale base with drifted backup and schema")
|
||||
stale_base: Final = _commit_code_only(tmp_path, "stale base with drifted backup")
|
||||
head: Final = _commit_code_only(tmp_path, "code change on the stale base")
|
||||
human: Final = _run_guard(tmp_path, stale_base, head, "litellm_fix_pricing")
|
||||
assert human.returncode == 0, human.stdout + human.stderr
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
|
|
@ -8,7 +7,6 @@ from pathlib import Path
|
|||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import jsonschema
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
|
|
@ -17,27 +15,8 @@ from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
|||
from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py"
|
||||
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
|
||||
|
||||
|
||||
def build_validator(schema: dict) -> jsonschema.Draft202012Validator:
|
||||
return jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER)
|
||||
|
||||
|
||||
def load_generator():
|
||||
spec = importlib.util.spec_from_file_location("generate_model_prices_schema", GENERATOR_PATH)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def committed_schema() -> dict:
|
||||
return json.loads(SCHEMA_PATH.read_text())
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
|
@ -45,183 +24,6 @@ def prices() -> dict:
|
|||
return json.loads(PRICES_PATH.read_text())
|
||||
|
||||
|
||||
def test_committed_schema_matches_generator_output(prices: dict, committed_schema: dict):
|
||||
generator = load_generator()
|
||||
regenerated = json.loads(generator.render(generator.build_schema(prices)))
|
||||
assert regenerated == committed_schema, (
|
||||
"model_prices_and_context_window.schema.json is out of sync; "
|
||||
"run `python ci_cd/generate_model_prices_schema.py` and commit the result"
|
||||
)
|
||||
|
||||
|
||||
def test_prices_file_validates_against_committed_schema(prices: dict, committed_schema: dict):
|
||||
validator = build_validator(committed_schema)
|
||||
errors = [
|
||||
f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
|
||||
for error in validator.iter_errors(prices)
|
||||
]
|
||||
assert errors == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"entry",
|
||||
[
|
||||
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": "0.01"},
|
||||
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": -1},
|
||||
{"litellm_provider": "openai", "mode": "not_a_real_mode"},
|
||||
{"mode": "chat"},
|
||||
{"litellm_provider": "openai", "deprecation_date": "June 2026"},
|
||||
{"litellm_provider": "openai", "deprecation_date": "2026-99-99"},
|
||||
{"litellm_provider": "openai", "deprecation_date": "2026-13-01"},
|
||||
{"litellm_provider": "openai", "deprecation_date": "2026-01-32"},
|
||||
{"litellm_provider": "openai", "deprecation_date": "2026-01-00"},
|
||||
{"litellm_provider": "openai", "deprecation_date": "2026-02-31"},
|
||||
{"litellm_provider": "openai", "supported_modalities": ["smell"]},
|
||||
{"litellm_provider": "openai", "supports_vision": "yes"},
|
||||
{"litellm_provider": "openai", "max_tokens": 8191.5},
|
||||
{"litellm_provider": "openai", "tiered_pricing": [{"unknown_tier_field": 1}]},
|
||||
],
|
||||
ids=[
|
||||
"cost_as_string",
|
||||
"negative_cost",
|
||||
"unknown_mode",
|
||||
"missing_provider",
|
||||
"non_iso_deprecation_date",
|
||||
"impossible_month_and_day",
|
||||
"month_out_of_range",
|
||||
"day_out_of_range",
|
||||
"day_zero",
|
||||
"calendar_impossible_day",
|
||||
"unknown_modality",
|
||||
"boolean_flag_as_string",
|
||||
"fractional_max_tokens",
|
||||
"unknown_tiered_pricing_field",
|
||||
],
|
||||
)
|
||||
def test_schema_rejects_malformed_entries(committed_schema: dict, entry: dict):
|
||||
validator = build_validator(committed_schema)
|
||||
assert not validator.is_valid({"some-model": entry})
|
||||
|
||||
|
||||
def test_schema_accepts_minimal_and_unknown_optional_fields(committed_schema: dict):
|
||||
validator = build_validator(committed_schema)
|
||||
assert validator.is_valid({"some-model": {"litellm_provider": "openai"}})
|
||||
assert validator.is_valid({"some-model": {"litellm_provider": "openai", "brand_new_field": {"nested": True}}})
|
||||
|
||||
|
||||
def test_schema_accepts_cache_creation_cost_inside_a_pricing_tier(committed_schema: dict):
|
||||
validator = build_validator(committed_schema)
|
||||
entry = {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "chat",
|
||||
"tiered_pricing": [
|
||||
{
|
||||
"range": [0, 256000],
|
||||
"input_cost_per_token": 3.25e-07,
|
||||
"output_cost_per_token": 1.95e-06,
|
||||
"cache_creation_input_token_cost": 4.063e-07,
|
||||
"cache_read_input_token_cost": 3.25e-08,
|
||||
}
|
||||
],
|
||||
}
|
||||
assert validator.is_valid({"some-model": entry})
|
||||
|
||||
|
||||
OFF_PEAK_ENTRY: Final = MappingProxyType(
|
||||
{
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 8e-6,
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "16:30-00:30",
|
||||
"windows": [{"hours_utc": ["00:30-02:00"], "weekdays": [6, "Sunday", "mon", "THURS"]}],
|
||||
"weekday_timezone": "Asia/Shanghai",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 4e-6,
|
||||
"cache_read_input_token_cost": 1e-7,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_generator_classifies_off_peak_pricing_as_a_windowed_rate_block():
|
||||
generator = load_generator()
|
||||
schema = json.loads(generator.render(generator.build_schema({"some-model": dict(OFF_PEAK_ENTRY)})))
|
||||
validator = build_validator(schema)
|
||||
assert validator.is_valid({"some-model": dict(OFF_PEAK_ENTRY)})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"block",
|
||||
[
|
||||
{"hours_utc": "16:30-00:30", "input_cost_per_token": "1e-6"},
|
||||
{"hours_utc": "16:30-00:30", "input_cost_per_token": -1e-6},
|
||||
{"hours_utc": 1630, "input_cost_per_token": 1e-6},
|
||||
{"hours_utc": "16:30-00:30", "discount": 0.5},
|
||||
{"windows": [{"weekdays": [6]}], "input_cost_per_token": 1e-6},
|
||||
{"windows": [{"hours_utc": "00:30-02:00", "weekdays": [0]}], "input_cost_per_token": 1e-6},
|
||||
{"windows": [], "input_cost_per_token": 1e-6},
|
||||
{"input_cost_per_token": 1e-6},
|
||||
{"hours_utc": "16:30", "input_cost_per_token": 1e-6},
|
||||
{"hours_utc": "25:00-01:00", "input_cost_per_token": 1e-6},
|
||||
{"hours_utc": ["16:30-00:30", "4pm-midnight"], "input_cost_per_token": 1e-6},
|
||||
{"windows": [{"hours_utc": "00:30-02:00", "weekdays": ["Funday"]}], "input_cost_per_token": 1e-6},
|
||||
],
|
||||
)
|
||||
def test_generated_off_peak_schema_rejects_malformed_blocks(block: dict):
|
||||
generator = load_generator()
|
||||
schema = json.loads(generator.render(generator.build_schema({"some-model": dict(OFF_PEAK_ENTRY)})))
|
||||
validator = build_validator(schema)
|
||||
assert not validator.is_valid({"some-model": {**OFF_PEAK_ENTRY, "off_peak_pricing": block}})
|
||||
|
||||
|
||||
TOKEN_COUNT_ENTRY: Final = MappingProxyType(
|
||||
{
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"mode": "chat",
|
||||
"token_count_framing": 78,
|
||||
"token_count_model": {"model": "us.anthropic.claude-opus-5-5", "region": "us-east-1"},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"setting",
|
||||
[
|
||||
{"token_count_framing": 0},
|
||||
{"token_count_framing": 78},
|
||||
{"token_count_model": {"model": "us.anthropic.claude-opus-5-5"}},
|
||||
],
|
||||
)
|
||||
def test_generator_classifies_token_count_settings(setting: dict):
|
||||
generator = load_generator()
|
||||
schema = json.loads(generator.render(generator.build_schema({"some-model": dict(TOKEN_COUNT_ENTRY)})))
|
||||
assert build_validator(schema).is_valid({"some-model": {**TOKEN_COUNT_ENTRY, **setting}})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"setting",
|
||||
[
|
||||
{"token_count_framing": -1},
|
||||
{"token_count_framing": "78"},
|
||||
{"token_count_framing": 1.5},
|
||||
{"token_count_model": {"model": ""}},
|
||||
{"token_count_model": {"model": " "}},
|
||||
{"token_count_model": {"model": "us.anthropic.claude-opus-5-5", "region": ""}},
|
||||
{"token_count_model": {"model": "us.anthropic.claude-opus-5-5", "region": " "}},
|
||||
{"token_count_model": {"model": "us.anthropic.claude-opus-5-5", "region": 5}},
|
||||
{"token_count_model": {"region": "us-east-1"}},
|
||||
{"token_count_model": "us.anthropic.claude-opus-5-5"},
|
||||
{"token_count_model": {"model": "us.anthropic.claude-opus-5-5", "zone": "us-east-1"}},
|
||||
],
|
||||
)
|
||||
def test_generated_token_count_schema_rejects_malformed_settings(setting: dict):
|
||||
generator = load_generator()
|
||||
schema = json.loads(generator.render(generator.build_schema({"some-model": dict(TOKEN_COUNT_ENTRY)})))
|
||||
assert not build_validator(schema).is_valid({"some-model": {**TOKEN_COUNT_ENTRY, **setting}})
|
||||
|
||||
|
||||
def find_duplicate_keys(path: Path) -> list[str]:
|
||||
duplicates: list[str] = []
|
||||
|
||||
Loading…
Add table
Reference in a new issue