From da5a9ffd2e25e7eecad646c1bc9771842331106a Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:49:48 -0700 Subject: [PATCH] ci(cost map): remove the generated model prices JSON schema (#47064) Backport of #47064 to stable/1.104.x. Cherry-picked from ae6f7f661355d0764031678d014c29f765897b68 (main). Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../auto_update_price_and_context_window.yml | 5 +- .github/workflows/sync-together-ai-models.yml | 7 +- ci_cd/cost_map_guard.py | 45 +- ci_cd/generate_model_prices_schema.py | 424 ------------------ litellm-rust/crates/model-catalog/AGENTS.md | 1 - tests/unit/test_cost_map_guard.py | 52 +-- ...> test_model_prices_and_context_window.py} | 198 -------- 7 files changed, 27 insertions(+), 705 deletions(-) delete mode 100644 ci_cd/generate_model_prices_schema.py rename tests/unit/{test_model_prices_schema.py => test_model_prices_and_context_window.py} (67%) diff --git a/.github/workflows/auto_update_price_and_context_window.yml b/.github/workflows/auto_update_price_and_context_window.yml index 7e40a860ee9..838ba35f831 100644 --- a/.github/workflows/auto_update_price_and_context_window.yml +++ b/.github/workflows/auto_update_price_and_context_window.yml @@ -24,12 +24,9 @@ jobs: - name: Update JSON Data run: | uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py" - - name: Regenerate JSON Schema - run: | - uv run --frozen python ci_cd/generate_model_prices_schema.py - name: Create Pull Request run: | - git add model_prices_and_context_window.json model_prices_and_context_window.schema.json + git add model_prices_and_context_window.json git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')" gh pr create --title "Update model_prices_and_context_window.json file" \ --body "Automated update for model_prices_and_context_window.json" \ diff --git a/.github/workflows/sync-together-ai-models.yml b/.github/workflows/sync-together-ai-models.yml index 18d5e3de1eb..5325510e45e 100644 --- a/.github/workflows/sync-together-ai-models.yml +++ b/.github/workflows/sync-together-ai-models.yml @@ -41,10 +41,6 @@ jobs: uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md" env: TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} - - name: Regenerate the JSON schema - if: steps.existing.outputs.open_pr == '' - run: | - uv run --frozen python ci_cd/generate_model_prices_schema.py - name: Create a pull request when the registry changed if: steps.existing.outputs.open_pr == '' run: | @@ -57,8 +53,7 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git checkout -b "$branch" git add model_prices_and_context_window.json \ - litellm/model_prices_and_context_window_backup.json \ - model_prices_and_context_window.schema.json + litellm/model_prices_and_context_window_backup.json git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')" gh auth setup-git git push origin "$branch" diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 14eecf01461..8e34e79a354 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -1,11 +1,11 @@ """Guard the cost map on pull requests. -Every pull request whose diff against its merge base touches one of the three cost map files gets the file -checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the -map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies -and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot -(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three -files and may only add or update models. +Every pull request whose diff against its merge base touches one of the two cost map files gets the file +checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched +skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the +branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always +get the file checks and additionally may only touch those two files or the legacy schema +path and may only add or update models. """ from __future__ import annotations @@ -19,13 +19,12 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Final -from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors - COST_MAP_PATH: Final = "model_prices_and_context_window.json" BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" -SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json" -GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH) +LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json" +GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" +SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"}) EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({}) CostMap = dict[str, object] @@ -35,7 +34,6 @@ CostMap = dict[str, object] class Snapshot: cost_map: str backup: str - schema: str def _parse_object(text: str, path: str) -> CostMap | str: @@ -46,13 +44,6 @@ def _parse_object(text: str, path: str) -> CostMap | str: return parsed if isinstance(parsed, dict) else f"{path} must be a JSON object at the root" -def _rendered_schema(cost_map: CostMap) -> str: - try: - return render(build_schema(cost_map)) - except SystemExit as error: - return str(error) - - def _evaluation_output_price_failure( key: str, entry: object, evaluation_output_price_sources: Mapping[str, str] ) -> str | None: @@ -78,29 +69,13 @@ def _file_failures( *, evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES, ) -> tuple[str, ...]: - schema_text: Final = _rendered_schema(head_map) - if not schema_text.startswith("{"): - return (schema_text,) backup_failure: Final = ( () if head.backup == head.cost_map else (f"{BACKUP_PATH} differs from {COST_MAP_PATH}; copy the root file over it",) ) - schema_failure: Final = ( - () - if head.schema == schema_text - else ( - f"{SCHEMA_PATH} is out of sync with {COST_MAP_PATH}; " - "run `python ci_cd/generate_model_prices_schema.py` and commit the result", - ) - ) return ( *backup_failure, - *schema_failure, - *( - f"{COST_MAP_PATH} does not validate against its schema: {error}" - for error in validation_errors(head_map, json.loads(schema_text))[:20] - ), *( failure for key, entry in head_map.items() @@ -135,7 +110,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str *( f"bot PRs may only change the cost map files, not {path}" for path in changed_files - if path not in GUARDED_PATHS + if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH ), *(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map), *(f"bot PRs may not remove fields: {ref}" for ref in removed_fields), diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py deleted file mode 100644 index 359df8e6e32..00000000000 --- a/ci_cd/generate_model_prices_schema.py +++ /dev/null @@ -1,424 +0,0 @@ -from __future__ import annotations - -import json -import sys -from pathlib import Path -from typing import Final, Optional - -import jsonschema - -REPO_ROOT = Path(__file__).parent.parent -PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" -SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json" - -SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"}) - -JsonSchema = dict - -NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0} -NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0} -BOOLEAN: JsonSchema = {"type": "boolean"} -STRING: JsonSchema = {"type": "string"} -NON_BLANK_STRING: Final[JsonSchema] = {"type": "string", "pattern": r"\S"} -TIME_WINDOW: Final[JsonSchema] = {"type": "string", "pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"} -WEEKDAY_PATTERN: Final = ( - r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$" -) - -EXTRA_BOOLEAN_KEYS = frozenset( - { - "gemini_native_audio", - "gemini_audio_only_live", - "uses_embed_content", - "use_openai_responses_path", - "bedrock_converse_supports_strict_tools", - "thinking_always_on", - } -) - -HOURS_UTC: Final[JsonSchema] = { - "description": 'UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight.', - "oneOf": [TIME_WINDOW, {"type": "array", "items": TIME_WINDOW, "minItems": 1}], -} - -OFF_PEAK_WINDOW: Final[JsonSchema] = { - "type": "object", - "properties": { - "hours_utc": HOURS_UTC, - "weekdays": { - "type": "array", - "description": "ISO-8601 weekday numbers (1 = Monday .. 7 = Sunday) or English day names the window applies on.", - "items": { - "oneOf": [ - {"type": "integer", "minimum": 1, "maximum": 7}, - {"type": "string", "pattern": WEEKDAY_PATTERN}, - ] - }, - "minItems": 1, - }, - }, - "required": ["hours_utc"], - "additionalProperties": False, -} - -OBJECT_KEYS: dict[str, JsonSchema] = { - "off_peak_pricing": { - "type": "object", - "description": "Rates that replace the same-named base fields while the request falls inside the stated UTC windows.", - "properties": { - "hours_utc": HOURS_UTC, - "windows": {"type": "array", "items": OFF_PEAK_WINDOW, "minItems": 1}, - "weekday_timezone": { - "type": "string", - "description": "IANA zone the weekdays of each window are read on; defaults to UTC.", - }, - "input_cost_per_token": NONNEG_NUMBER, - "output_cost_per_token": NONNEG_NUMBER, - "output_cost_per_reasoning_token": NONNEG_NUMBER, - "cache_read_input_token_cost": NONNEG_NUMBER, - "cache_creation_input_token_cost": NONNEG_NUMBER, - }, - "anyOf": [{"required": ["hours_utc"]}, {"required": ["windows"]}], - "additionalProperties": False, - }, - "search_context_cost_per_query": { - "type": "object", - "description": "USD cost per web search query, keyed by search context size.", - "properties": { - "search_context_size_low": NONNEG_NUMBER, - "search_context_size_medium": NONNEG_NUMBER, - "search_context_size_high": NONNEG_NUMBER, - }, - "additionalProperties": False, - }, - "guardrail_cost_per_unit": { - "type": "object", - "description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).", - "additionalProperties": NONNEG_NUMBER, - }, - "metadata": { - "type": "object", - "description": "Free-form notes about the entry (e.g. pricing derivation).", - }, - "provider_specific_entry": { - "type": "object", - "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).", - }, - "token_count_model": { - "type": "object", - "required": ["model"], - "additionalProperties": False, - "properties": { - "model": NON_BLANK_STRING, - "region": NON_BLANK_STRING, - }, - "description": "Model that counts tokens for this one, and the AWS region its count API runs in.", - }, -} - -ARRAY_KEYS: dict[str, JsonSchema] = { - "supported_audio_formats": { - "type": "array", - "description": "Audio container formats the model can return.", - "items": {"type": "string", "enum": ["mp3", "wav"]}, - }, - "supported_endpoints": { - "type": "array", - "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.", - "items": STRING, - }, - "supported_modalities": { - "type": "array", - "description": "Input modalities the model accepts.", - "items": {"type": "string", "enum": ["text", "image", "audio", "video"]}, - }, - "supported_output_modalities": { - "type": "array", - "description": "Output modalities the model can produce.", - "items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]}, - }, - "reasoning_effort_levels": { - "type": "array", - "description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.", - "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]}, - }, - "supported_regions": { - "type": "array", - "description": "Cloud regions the model is available in ('global' or region ids).", - "items": STRING, - }, - "tiered_pricing": { - "type": "array", - "description": "Context-length or result-count tiered rates; each tier's costs apply within its range.", - "items": { - "type": "object", - "properties": { - "range": { - "type": "array", - "description": "[min, max] prompt-token span this tier applies to.", - "items": NONNEG_NUMBER, - "minItems": 2, - "maxItems": 2, - }, - "max_results_range": { - "type": "array", - "description": "[min, max] result-count span this tier applies to (search models).", - "items": NONNEG_NUMBER, - "minItems": 2, - "maxItems": 2, - }, - "input_cost_per_token": NONNEG_NUMBER, - "output_cost_per_token": NONNEG_NUMBER, - "output_cost_per_reasoning_token": NONNEG_NUMBER, - "cache_read_input_token_cost": NONNEG_NUMBER, - "cache_creation_input_token_cost": NONNEG_NUMBER, - "input_cost_per_query": NONNEG_NUMBER, - }, - "additionalProperties": False, - }, - }, -} - -INTEGER_KEYS: dict[str, JsonSchema] = { - "max_tokens": { - **NONNEG_INTEGER, - "description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.", - }, - "max_input_tokens": { - **NONNEG_INTEGER, - "description": "Maximum prompt/context tokens the model accepts.", - }, - "max_output_tokens": { - **NONNEG_INTEGER, - "description": "Maximum tokens the model can generate in one response.", - }, - "output_vector_size": { - **NONNEG_INTEGER, - "description": "Embedding dimension for embedding models.", - }, - "prompt_cache_min_tokens": { - **NONNEG_INTEGER, - "description": "Smallest prefix the provider will actually cache; absent means the provider default applies.", - }, - "token_count_framing": { - **NONNEG_INTEGER, - "description": "Tokens added once per request to a local count estimate, for the provider's chat framing.", - }, - "tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."}, - "rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."}, -} - -NUMBER_KEYS: dict[str, JsonSchema] = { - "regional_processing_uplift_multiplier_eu": { - "type": "number", - "minimum": 1, - "description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).", - }, - "regional_processing_uplift_multiplier_us": { - "type": "number", - "minimum": 1, - "description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).", - }, - "regional_endpoint_uplift_multiplier": { - "type": "number", - "minimum": 1, - "description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).", - }, -} - -COST_DESCRIPTIONS: dict[str, str] = { - "input_cost_per_token": "USD per prompt token.", - "output_cost_per_token": "USD per generated token.", - "output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.", - "google_maps_grounding_cost_per_query": ( - "USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit." - ), - "cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.", - "cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.", - "input_cost_per_token_batches": "USD per prompt token via the provider's batch API.", - "output_cost_per_token_batches": "USD per generated token via the provider's batch API.", -} - - -def cost_description(key: str) -> Optional[str]: - if key in COST_DESCRIPTIONS: - return COST_DESCRIPTIONS[key] - if key.endswith("_flex"): - return "Flex service-tier rate for the same-named base field." - if key.endswith("_priority"): - return "Priority service-tier rate for the same-named base field." - if "_above_" in key: - return "Rate applied once the prompt exceeds the token threshold in the field name." - return None - - -def cost_schema(key: str) -> JsonSchema: - description = cost_description(key) - return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER) - - -def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]: - return { - "litellm_provider": { - "type": "string", - "description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.", - }, - "mode": { - "type": "string", - "description": "Primary API surface / task type of the model.", - "enum": list(modes), - }, - "source": { - "type": "string", - "description": "URL of the provider pricing/model page this entry was taken from.", - }, - "deprecation_date": { - "type": "string", - "description": "Date the provider deprecates the model, YYYY-MM-DD.", - "format": "date", - "pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$", - }, - "web_search_billing_unit": { - "type": "string", - "description": "Whether web search is billed per query or per prompt.", - "enum": ["per_query", "per_prompt"], - }, - "bedrock_output_config_effort_ceiling": { - "type": "string", - "description": "Highest reasoning effort the Bedrock output_config accepts for this model.", - "enum": ["low", "medium", "high", "max", "xhigh"], - }, - "default_reasoning_effort": { - "type": "string", - "description": ( - "Reasoning effort the provider applies when the request omits reasoning_effort. " - "Gates whether a non-default temperature or the top_p/logprobs sampling params are " - "accepted, which hold only when the effort resolves to 'none'." - ), - "enum": ["none", "minimal", "low", "medium", "high", "xhigh"], - }, - "comment": STRING, - "audio_transcription_config": STRING, - "vertex_ai_audio_api": { - "type": "string", - "enum": ["lyria_predict", "lyria_interactions"], - }, - } - - -def classify(key: str, modes: tuple) -> Optional[JsonSchema]: - curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS} - if key in curated: - return curated[key] - if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS: - return BOOLEAN - if "cost" in key: - return cost_schema(key) - return None - - -def build_schema(prices: dict) -> JsonSchema: - entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS} - all_keys = tuple(sorted({key for entry in entries.values() for key in entry})) - modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry})) - unclassified = tuple(key for key in all_keys if classify(key, modes) is None) - if unclassified: - raise SystemExit( - f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. " - f"Add them to the key tables in {Path(__file__).name} and rerun it." - ) - entry_properties = {key: classify(key, modes) for key in all_keys} - return { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "LiteLLM model_prices_and_context_window.json", - "description": ( - "Schema for LiteLLM's model price and context window registry " - "(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). " - "Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, " - "optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. " - "All costs are USD per unit. New optional fields are added regularly, so consumers should " - "ignore unknown fields rather than reject them." - ), - "type": "object", - "properties": { - "sample_spec": { - "type": "object", - "description": ( - "Documentation placeholder illustrating the entry shape; not a real model and not " - "schema-conformant (several values are prose)." - ), - }, - "fallback_generalizations": { - "type": "object", - "description": "Regex rules that generalize unknown model ids to known families; not a model entry.", - "properties": { - "rules": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": STRING, - "pattern": STRING, - "description": STRING, - }, - "required": ["name", "pattern"], - "additionalProperties": True, - }, - } - }, - "additionalProperties": False, - }, - }, - "additionalProperties": {"$ref": "#/$defs/modelEntry"}, - "$defs": { - "modelEntry": { - "type": "object", - "description": ( - "Pricing, limits, and capability flags for one model. Fields other than litellm_provider " - "are optional; boolean capability flags are simply omitted when unknown or false." - ), - "required": ["litellm_provider"], - "properties": entry_properties, - "additionalProperties": True, - } - }, - } - - -def render(schema: JsonSchema) -> str: - return json.dumps(schema, indent=2) + "\n" - - -def validation_errors(prices: dict, schema: JsonSchema) -> tuple: - validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER) - return tuple( - f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}" - for error in validator.iter_errors(prices) - ) - - -def main() -> int: - check = "--check" in sys.argv[1:] - prices = json.loads(PRICES_PATH.read_text()) - rendered = render(build_schema(prices)) - errors = validation_errors(prices, json.loads(rendered)) - if errors: - print(f"{PRICES_PATH.name} does not validate against the generated schema:") - print("\n".join(errors[:20])) - return 1 - if not check: - SCHEMA_PATH.write_text(rendered) - print(f"wrote {SCHEMA_PATH}") - return 0 - if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered: - print( - f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. " - f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result." - ) - return 1 - print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/litellm-rust/crates/model-catalog/AGENTS.md b/litellm-rust/crates/model-catalog/AGENTS.md index 9fcbcd57a76..0a380788c19 100644 --- a/litellm-rust/crates/model-catalog/AGENTS.md +++ b/litellm-rust/crates/model-catalog/AGENTS.md @@ -2,5 +2,4 @@ For `model_prices_and_context_window.json` validation, we should eventually: -- Remove any schema file like `model_prices_and_context_window.schema.json` - Stop skipping this crate's tests diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index e2c54d381d8..410b3acfd42 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -22,7 +22,6 @@ def _load(name: str) -> ModuleType: return module -schema_module: Final = _load("generate_model_prices_schema") guard: Final = _load("cost_map_guard") MAP_FILES: Final = (guard.COST_MAP_PATH,) @@ -52,12 +51,9 @@ def _serialize(cost_map: dict[str, object]) -> str: return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n" -def _snapshot(cost_map: dict[str, object], backup: str | None = None, schema: str | None = None) -> object: +def _snapshot(cost_map: dict[str, object], backup: str | None = None) -> object: text = _serialize(cost_map) - rendered = schema_module.render(schema_module.build_schema(cost_map)) - return guard.Snapshot( - cost_map=text, backup=text if backup is None else backup, schema=rendered if schema is None else schema - ) + return guard.Snapshot(cost_map=text, backup=text if backup is None else backup) BASE: Final = _snapshot(BASE_MAP) @@ -135,14 +131,14 @@ def test_bot_may_add_and_reprice_models() -> None: def test_broken_json_is_reported() -> None: - head = guard.Snapshot(cost_map="{not json", backup="{not json", schema="{}") + head = guard.Snapshot(cost_map="{not json", backup="{not json") assert _failures(head, bot=False) == ( f"{guard.COST_MAP_PATH} is not valid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)", ) def test_non_object_root_is_reported() -> None: - head = guard.Snapshot(cost_map="[]", backup="[]", schema="{}") + head = guard.Snapshot(cost_map="[]", backup="[]") assert _failures(head, bot=False) == (f"{guard.COST_MAP_PATH} must be a JSON object at the root",) @@ -151,28 +147,7 @@ def test_backup_drift_is_reported() -> None: assert [failure for failure in _failures(head, bot=False) if failure.startswith(guard.BACKUP_PATH)] -def test_schema_out_of_sync_is_reported() -> None: - head = _snapshot({**BASE_MAP, "openrouter/c": _entry(supports_audio_input=True)}, schema=BASE.schema) - assert [failure for failure in _failures(head, bot=False) if failure.startswith(guard.SCHEMA_PATH)] - - -def test_schema_validation_errors_are_reported() -> None: - head = _snapshot({**BASE_MAP, "openrouter/c": _entry(-1e-06)}) - prefix = f"{guard.COST_MAP_PATH} does not validate against its schema: openrouter/c." - assert [failure.removeprefix(prefix).split(":")[0] for failure in _failures(head, bot=False)] == [ - "input_cost_per_token", - "output_cost_per_token", - ] - - -def test_unclassified_entry_key_is_reported() -> None: - text = _serialize({**BASE_MAP, "openrouter/c": _entry(weird_thing=1)}) - head = guard.Snapshot(cost_map=text, backup=text, schema=BASE.schema) - (failure,) = _failures(head, bot=False) - assert "Unclassified keys" in failure and "weird_thing" in failure - - -STALE_HEAD: Final = _snapshot(BASE_MAP, backup=_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)}), schema="{}") +STALE_HEAD: Final = _snapshot(BASE_MAP, backup=_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)})) CODE_ONLY: Final = ( "litellm/utils.py", "tests/test_litellm/test_utils.py", @@ -181,7 +156,7 @@ CODE_ONLY: Final = ( def test_human_pr_that_leaves_the_cost_map_alone_skips_the_file_checks() -> None: - unparseable: Final = guard.Snapshot(cost_map="{not json", backup="", schema="") + unparseable: Final = guard.Snapshot(cost_map="{not json", backup="") assert _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=False) == () assert _failures(STALE_HEAD, changed_files=(), bot=False) == () assert _failures(unparseable, changed_files=CODE_ONLY, bot=False) == () @@ -191,18 +166,16 @@ def test_human_pr_that_leaves_the_cost_map_alone_skips_the_file_checks() -> None def test_touching_any_cost_map_file_keeps_the_file_checks(guarded_path: str) -> None: failures: Final = _failures(STALE_HEAD, changed_files=(*CODE_ONLY, guarded_path), bot=False) assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)] - assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)] def test_bot_pr_always_gets_the_file_checks() -> None: failures: Final = _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=True) assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)] - assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)] def test_contract_names_the_skip() -> None: assert guard.contract_for(False, CODE_ONLY) == "human PR, cost map untouched" - assert guard.contract_for(False, (*CODE_ONLY, guard.SCHEMA_PATH)) == "human PR, file checks only" + assert guard.contract_for(False, (*CODE_ONLY, guard.COST_MAP_PATH)) == "human PR, file checks only" assert guard.contract_for(True, CODE_ONLY) == "bot contract enforced" @@ -215,6 +188,13 @@ def test_bot_may_only_touch_the_cost_map_files() -> None: ) +def test_bot_may_update_cost_map_and_legacy_schema() -> None: + head: Final = _snapshot({**BASE_MAP, "openrouter/c": _entry()}) + changed: Final = (*guard.GUARDED_PATHS, guard.LEGACY_SCHEMA_PATH) + + assert _failures(head, changed_files=changed) == () + + def test_bot_may_not_remove_models() -> None: head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"}) assert _failures(head, bot=False) == () @@ -238,7 +218,6 @@ def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str: (repo / guard.COST_MAP_PATH).write_text(text) (repo / guard.BACKUP_PATH).parent.mkdir(exist_ok=True) (repo / guard.BACKUP_PATH).write_text(text) - (repo / guard.SCHEMA_PATH).write_text(schema_module.render(schema_module.build_schema(cost_map))) return _git_commit(repo, message) @@ -292,8 +271,7 @@ def test_main_skips_the_file_checks_on_a_stale_base_the_pr_never_touched(tmp_pat subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) _commit(tmp_path, BASE_MAP, "base") (tmp_path / guard.BACKUP_PATH).write_text(_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)})) - (tmp_path / guard.SCHEMA_PATH).write_text("{}") - stale_base: Final = _commit_code_only(tmp_path, "stale base with drifted backup and schema") + stale_base: Final = _commit_code_only(tmp_path, "stale base with drifted backup") head: Final = _commit_code_only(tmp_path, "code change on the stale base") human: Final = _run_guard(tmp_path, stale_base, head, "litellm_fix_pricing") assert human.returncode == 0, human.stdout + human.stderr diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_and_context_window.py similarity index 67% rename from tests/unit/test_model_prices_schema.py rename to tests/unit/test_model_prices_and_context_window.py index 211a43d2574..49a6fb05b15 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -1,6 +1,5 @@ from __future__ import annotations -import importlib.util import json import re from collections.abc import Mapping @@ -8,7 +7,6 @@ from pathlib import Path from types import MappingProxyType from typing import Final -import jsonschema import pytest import litellm @@ -17,27 +15,8 @@ from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts REPO_ROOT = Path(__file__).parents[2] -GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py" PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" -SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json" - - -def build_validator(schema: dict) -> jsonschema.Draft202012Validator: - return jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER) - - -def load_generator(): - spec = importlib.util.spec_from_file_location("generate_model_prices_schema", GENERATOR_PATH) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -@pytest.fixture(scope="module") -def committed_schema() -> dict: - return json.loads(SCHEMA_PATH.read_text()) @pytest.fixture(scope="module") @@ -45,183 +24,6 @@ def prices() -> dict: return json.loads(PRICES_PATH.read_text()) -def test_committed_schema_matches_generator_output(prices: dict, committed_schema: dict): - generator = load_generator() - regenerated = json.loads(generator.render(generator.build_schema(prices))) - assert regenerated == committed_schema, ( - "model_prices_and_context_window.schema.json is out of sync; " - "run `python ci_cd/generate_model_prices_schema.py` and commit the result" - ) - - -def test_prices_file_validates_against_committed_schema(prices: dict, committed_schema: dict): - validator = build_validator(committed_schema) - errors = [ - f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}" - for error in validator.iter_errors(prices) - ] - assert errors == [] - - -@pytest.mark.parametrize( - "entry", - [ - {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": "0.01"}, - {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": -1}, - {"litellm_provider": "openai", "mode": "not_a_real_mode"}, - {"mode": "chat"}, - {"litellm_provider": "openai", "deprecation_date": "June 2026"}, - {"litellm_provider": "openai", "deprecation_date": "2026-99-99"}, - {"litellm_provider": "openai", "deprecation_date": "2026-13-01"}, - {"litellm_provider": "openai", "deprecation_date": "2026-01-32"}, - {"litellm_provider": "openai", "deprecation_date": "2026-01-00"}, - {"litellm_provider": "openai", "deprecation_date": "2026-02-31"}, - {"litellm_provider": "openai", "supported_modalities": ["smell"]}, - {"litellm_provider": "openai", "supports_vision": "yes"}, - {"litellm_provider": "openai", "max_tokens": 8191.5}, - {"litellm_provider": "openai", "tiered_pricing": [{"unknown_tier_field": 1}]}, - ], - ids=[ - "cost_as_string", - "negative_cost", - "unknown_mode", - "missing_provider", - "non_iso_deprecation_date", - "impossible_month_and_day", - "month_out_of_range", - "day_out_of_range", - "day_zero", - "calendar_impossible_day", - "unknown_modality", - "boolean_flag_as_string", - "fractional_max_tokens", - "unknown_tiered_pricing_field", - ], -) -def test_schema_rejects_malformed_entries(committed_schema: dict, entry: dict): - validator = build_validator(committed_schema) - assert not validator.is_valid({"some-model": entry}) - - -def test_schema_accepts_minimal_and_unknown_optional_fields(committed_schema: dict): - validator = build_validator(committed_schema) - assert validator.is_valid({"some-model": {"litellm_provider": "openai"}}) - assert validator.is_valid({"some-model": {"litellm_provider": "openai", "brand_new_field": {"nested": True}}}) - - -def test_schema_accepts_cache_creation_cost_inside_a_pricing_tier(committed_schema: dict): - validator = build_validator(committed_schema) - entry = { - "litellm_provider": "dashscope", - "mode": "chat", - "tiered_pricing": [ - { - "range": [0, 256000], - "input_cost_per_token": 3.25e-07, - "output_cost_per_token": 1.95e-06, - "cache_creation_input_token_cost": 4.063e-07, - "cache_read_input_token_cost": 3.25e-08, - } - ], - } - assert validator.is_valid({"some-model": entry}) - - -OFF_PEAK_ENTRY: Final = MappingProxyType( - { - "litellm_provider": "openrouter", - "mode": "chat", - "input_cost_per_token": 2e-6, - "output_cost_per_token": 8e-6, - "off_peak_pricing": { - "hours_utc": "16:30-00:30", - "windows": [{"hours_utc": ["00:30-02:00"], "weekdays": [6, "Sunday", "mon", "THURS"]}], - "weekday_timezone": "Asia/Shanghai", - "input_cost_per_token": 1e-6, - "output_cost_per_token": 4e-6, - "cache_read_input_token_cost": 1e-7, - }, - } -) - - -def test_generator_classifies_off_peak_pricing_as_a_windowed_rate_block(): - generator = load_generator() - schema = json.loads(generator.render(generator.build_schema({"some-model": dict(OFF_PEAK_ENTRY)}))) - validator = build_validator(schema) - assert validator.is_valid({"some-model": dict(OFF_PEAK_ENTRY)}) - - -@pytest.mark.parametrize( - "block", - [ - {"hours_utc": "16:30-00:30", "input_cost_per_token": "1e-6"}, - {"hours_utc": "16:30-00:30", "input_cost_per_token": -1e-6}, - {"hours_utc": 1630, "input_cost_per_token": 1e-6}, - {"hours_utc": "16:30-00:30", "discount": 0.5}, - {"windows": [{"weekdays": [6]}], "input_cost_per_token": 1e-6}, - {"windows": [{"hours_utc": "00:30-02:00", "weekdays": [0]}], "input_cost_per_token": 1e-6}, - {"windows": [], "input_cost_per_token": 1e-6}, - {"input_cost_per_token": 1e-6}, - {"hours_utc": "16:30", "input_cost_per_token": 1e-6}, - {"hours_utc": "25:00-01:00", "input_cost_per_token": 1e-6}, - {"hours_utc": ["16:30-00:30", "4pm-midnight"], "input_cost_per_token": 1e-6}, - {"windows": [{"hours_utc": "00:30-02:00", "weekdays": ["Funday"]}], "input_cost_per_token": 1e-6}, - ], -) -def test_generated_off_peak_schema_rejects_malformed_blocks(block: dict): - generator = load_generator() - schema = json.loads(generator.render(generator.build_schema({"some-model": dict(OFF_PEAK_ENTRY)}))) - validator = build_validator(schema) - assert not validator.is_valid({"some-model": {**OFF_PEAK_ENTRY, "off_peak_pricing": block}}) - - -TOKEN_COUNT_ENTRY: Final = MappingProxyType( - { - "litellm_provider": "bedrock_converse", - "mode": "chat", - "token_count_framing": 78, - "token_count_model": {"model": "us.anthropic.claude-opus-5-5", "region": "us-east-1"}, - } -) - - -@pytest.mark.parametrize( - "setting", - [ - {"token_count_framing": 0}, - {"token_count_framing": 78}, - {"token_count_model": {"model": "us.anthropic.claude-opus-5-5"}}, - ], -) -def test_generator_classifies_token_count_settings(setting: dict): - generator = load_generator() - schema = json.loads(generator.render(generator.build_schema({"some-model": dict(TOKEN_COUNT_ENTRY)}))) - assert build_validator(schema).is_valid({"some-model": {**TOKEN_COUNT_ENTRY, **setting}}) - - -@pytest.mark.parametrize( - "setting", - [ - {"token_count_framing": -1}, - {"token_count_framing": "78"}, - {"token_count_framing": 1.5}, - {"token_count_model": {"model": ""}}, - {"token_count_model": {"model": " "}}, - {"token_count_model": {"model": "us.anthropic.claude-opus-5-5", "region": ""}}, - {"token_count_model": {"model": "us.anthropic.claude-opus-5-5", "region": " "}}, - {"token_count_model": {"model": "us.anthropic.claude-opus-5-5", "region": 5}}, - {"token_count_model": {"region": "us-east-1"}}, - {"token_count_model": "us.anthropic.claude-opus-5-5"}, - {"token_count_model": {"model": "us.anthropic.claude-opus-5-5", "zone": "us-east-1"}}, - ], -) -def test_generated_token_count_schema_rejects_malformed_settings(setting: dict): - generator = load_generator() - schema = json.loads(generator.render(generator.build_schema({"some-model": dict(TOKEN_COUNT_ENTRY)}))) - assert not build_validator(schema).is_valid({"some-model": {**TOKEN_COUNT_ENTRY, **setting}}) - - def find_duplicate_keys(path: Path) -> list[str]: duplicates: list[str] = []