From 9041768fb43715dc8c28e5dc139c86adac2659ce Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:47:51 -0700 Subject: [PATCH] feat(cost_map): derive source_revision from the loaded bytes instead of a _metadata stamp The revision an operator checks is now the git blob id of the exact bytes the process loaded, the same id git rev-parse :model_prices_and_context_window.json prints, so it is always present, never goes stale between bot writes, and needs no stamp in the JSON that every PR touching the file would have to regenerate. The _metadata block, the generated_at field, the schema and guard changes, and the bot stamping are dropped --- ...to_update_price_and_context_window_file.py | 27 +--- ci_cd/cost_map_guard.py | 7 +- ci_cd/generate_model_prices_schema.py | 19 +-- .../litellm_core_utils/get_model_cost_map.py | 98 ++++++------- ...odel_prices_and_context_window_backup.json | 4 - litellm/proxy/proxy_server.py | 2 +- model_prices_and_context_window.json | 4 - model_prices_and_context_window.schema.json | 20 +-- scripts/sync_together_ai_models.py | 23 +-- .../test_get_model_cost_map.py | 135 +++++++----------- .../test_routes_model_cost_map.py | 24 ++-- ...to_update_price_and_context_window_file.py | 54 ------- tests/test_litellm/test_cost_map_guard.py | 20 --- .../test_litellm/test_model_prices_schema.py | 19 --- .../test_sync_together_ai_models.py | 53 ------- tests/test_litellm/test_utils.py | 17 ++- .../src/components/price_data_reload.test.tsx | 15 +- .../src/components/price_data_reload.tsx | 8 -- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 19 files changed, 131 insertions(+), 420 deletions(-) delete mode 100644 tests/test_litellm/test_auto_update_price_and_context_window_file.py diff --git a/.github/scripts/auto_update_price_and_context_window_file.py b/.github/scripts/auto_update_price_and_context_window_file.py index a7a3194f262..461d8d347d9 100644 --- a/.github/scripts/auto_update_price_and_context_window_file.py +++ b/.github/scripts/auto_update_price_and_context_window_file.py @@ -1,9 +1,6 @@ import asyncio import aiohttp import json -import os -import subprocess -from datetime import datetime, timezone # Asynchronously fetch data from a given URL async def fetch_data(url): @@ -34,28 +31,13 @@ def sync_local_data_with_remote(local_data, remote_data): for key in (set(remote_data) - set(local_data)): local_data[key] = remote_data[key] -def utc_now_iso(): - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def source_revision(): - from_env = os.environ.get("GITHUB_SHA") - if from_env: - return from_env - return subprocess.run(["git", "rev-parse", "HEAD"], check=True, capture_output=True, text=True).stdout.strip() - - -def stamp_metadata(data, generated_at, revision): - return {**data, "_metadata": {"generated_at": generated_at, "source_revision": revision}} - - # Write data to the json file def write_to_file(file_path, data): try: # Open the file in write mode with open(file_path, "w") as file: # Dump the data as JSON into the file - file.write(json.dumps(data, indent=4) + "\n") + json.dump(data, file, indent=4) print("Values updated successfully.") except Exception as e: # Print an error message if writing to file fails @@ -167,13 +149,8 @@ def main(): # If both local and openrouter data are available, synchronize and save if local_data and all_remote_data: - before = json.dumps(local_data, sort_keys=True) sync_local_data_with_remote(local_data, all_remote_data) - changed = json.dumps(local_data, sort_keys=True) != before - write_to_file( - local_file_path, - stamp_metadata(local_data, utc_now_iso(), source_revision()) if changed else local_data, - ) + write_to_file(local_file_path, local_data) else: print("Failed to fetch model data from either local file or URL.") diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 351c06c74eb..50aa40ba220 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -2,8 +2,7 @@ Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named -litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models, plus -restamp the _metadata provenance block. +litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models. """ from __future__ import annotations @@ -16,7 +15,7 @@ from collections.abc import Sequence from dataclasses import dataclass from typing import Final -from generate_model_prices_schema import BOT_LOCKED_ROOT_KEYS, build_schema, render, validation_errors +from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors COST_MAP_PATH: Final = "model_prices_and_context_window.json" BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" @@ -103,7 +102,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str *(f"bot PRs may not remove fields: {ref}" for ref in removed_fields), *( f"bot PRs may not change {key}" - for key in sorted(BOT_LOCKED_ROOT_KEYS) + for key in sorted(SPECIAL_ROOT_KEYS) if base_map.get(key) != head_map.get(key) ), ) diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 557afa50128..ab29b70bdd4 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -11,9 +11,7 @@ REPO_ROOT = Path(__file__).parent.parent PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json" -METADATA_KEY = "_metadata" -SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations", METADATA_KEY}) -BOT_LOCKED_ROOT_KEYS = SPECIAL_ROOT_KEYS - {METADATA_KEY} +SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"}) JsonSchema = dict @@ -273,26 +271,13 @@ def build_schema(prices: dict) -> JsonSchema: "description": ( "Schema for LiteLLM's model price and context window registry " "(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). " - "Every top-level key except '_metadata', 'sample_spec', and 'fallback_generalizations' is a model id, " + "Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, " "optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. " "All costs are USD per unit. New optional fields are added regularly, so consumers should " "ignore unknown fields rather than reject them." ), "type": "object", "properties": { - METADATA_KEY: { - "type": "object", - "description": ( - "Provenance of this file: when an automated sync last regenerated it and the commit it " - "ran against. Human edits leave it untouched; not a model entry." - ), - "properties": { - "generated_at": {"type": "string", "format": "date-time"}, - "source_revision": STRING, - }, - "required": ["generated_at", "source_revision"], - "additionalProperties": False, - }, "sample_spec": { "type": "object", "description": ( diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index a538e7cb330..2bdfbc66088 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -9,18 +9,18 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True """ import asyncio +import hashlib import json import os import random import time from collections.abc import Awaitable, Callable -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime, timezone from importlib.resources import files from typing import Final, Protocol import httpx -from pydantic import BaseModel, ConfigDict, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm import verbose_logger @@ -33,11 +33,10 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" -METADATA_KEY: Final = "_metadata" # Reserved top-level keys that are not model entries. They must be excluded # from the model-count integrity check so a real upstream shrink can't be masked. -RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY, METADATA_KEY}) +RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY}) def _count_model_entries(model_cost: dict) -> int: @@ -45,6 +44,11 @@ def _count_model_entries(model_cost: dict) -> int: return sum(1 for key in model_cost if key not in RESERVED_TOP_LEVEL_KEYS) +def git_blob_id(body: bytes) -> str: + """The sha1 git gives these bytes as a blob, so ``git rev-parse :`` reproduces it for the file""" + return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest() + + class GetModelCostMap: """ Handles fetching, validating, and loading the model cost map. @@ -56,15 +60,25 @@ class GetModelCostMap: _backup_model_count: int = -1 # -1 = not yet loaded + @staticmethod + def read_local_model_cost_map_bytes() -> bytes: + return files("litellm").joinpath("model_prices_and_context_window_backup.json").read_bytes() + @staticmethod def read_local_model_cost_map_text() -> str: - return files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text(encoding="utf-8") + return GetModelCostMap.read_local_model_cost_map_bytes().decode("utf-8") + + @staticmethod + def load_local_model_cost_map_with_revision() -> "ModelCostMapReloaded": + """The bundled backup map together with the git blob id of the file it was parsed from""" + body: Final = GetModelCostMap.read_local_model_cost_map_bytes() + content: Final = json.loads(body) + return ModelCostMapReloaded(model_cost_map=content, revision=git_blob_id(body)) @staticmethod def load_local_model_cost_map() -> dict: """Load the local backup model cost map bundled with the package.""" - content: Final = json.loads(GetModelCostMap.read_local_model_cost_map_text()) - return content + return GetModelCostMap.load_local_model_cost_map_with_revision().model_cost_map @classmethod def _get_backup_model_count(cls) -> int: @@ -169,6 +183,7 @@ MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0 @dataclass(frozen=True, slots=True) class ModelCostMapReloaded: model_cost_map: dict # mutable-ok: adopted as litellm.model_cost, whose consumer contract is a plain mutable dict + revision: str | None = None etag: str | None = None @@ -258,7 +273,9 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}") if not isinstance(parsed, dict): return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}") - return ModelCostMapReloaded(model_cost_map=parsed, etag=response.headers.get("etag")) + return ModelCostMapReloaded( + model_cost_map=parsed, revision=git_blob_id(response.content), etag=response.headers.get("etag") + ) def _next_retry_wait( @@ -337,10 +354,7 @@ async def refetch_model_cost_map( _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True _cost_map_source_info.fallback_reason = None - _cost_map_source_info.etag = None - return ModelCostMapReloaded( - model_cost_map=_finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) - ) + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()) result: Final = await _fetch_remote_model_cost_map_with_retry( url=url, @@ -366,8 +380,7 @@ async def refetch_model_cost_map( _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False _cost_map_source_info.fallback_reason = None - _cost_map_source_info.etag = result.etag - return ModelCostMapReloaded(model_cost_map=_finalize_model_cost_map(result.model_cost_map), etag=result.etag) + return _finalize_loaded_model_cost_map(result) class ModelCostMapSourceInfo: @@ -378,7 +391,6 @@ class ModelCostMapSourceInfo: is_env_forced: bool = False fallback_reason: str | None = None loaded_at: "datetime | None" = None - generated_at: str | None = None source_revision: str | None = None etag: str | None = None @@ -387,28 +399,7 @@ class ModelCostMapSourceInfo: _cost_map_source_info: Final = ModelCostMapSourceInfo() -class CostMapMetadata(BaseModel): - model_config = ConfigDict(frozen=True, extra="ignore") - - generated_at: str | None = None - source_revision: str | None = None - - -_EMPTY_METADATA: Final = CostMapMetadata() - - -def _parse_metadata(raw: object) -> CostMapMetadata: - if raw is None: - return _EMPTY_METADATA - try: - return CostMapMetadata.model_validate(raw) - except ValidationError as error: - verbose_logger.warning("LiteLLM: ignoring a malformed %s block in the model cost map: %s", METADATA_KEY, error) - return _EMPTY_METADATA - - class CostMapProvenance(TypedDict): - generated_at: ReadOnly[str | None] source_revision: ReadOnly[str | None] etag: ReadOnly[str | None] @@ -422,10 +413,10 @@ class CostMapSourceInfo(CostMapProvenance): def get_model_cost_map_provenance() -> CostMapProvenance: - """Which revision of the cost map this process serves: the ``_metadata`` stamp the file - carries plus the ETag the remote fetch returned (None for the bundled backup)""" + """Which revision of the cost map this process serves: the git blob id of the bytes it loaded, the + same id ``git rev-parse :model_prices_and_context_window.json`` prints for a checkout, plus + the ETag the remote fetch returned (None for the bundled backup)""" return { - "generated_at": _cost_map_source_info.generated_at, "source_revision": _cost_map_source_info.source_revision, "etag": _cost_map_source_info.etag, } @@ -441,7 +432,7 @@ def get_model_cost_map_source_info() -> CostMapSourceInfo: - is_env_forced: True if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage - fallback_reason: human-readable reason if remote failed and local was used - loaded_at: ISO 8601 time this process last loaded the map - - generated_at, source_revision: the ``_metadata`` stamp inside the loaded file + - source_revision: git blob id of the loaded file's bytes - etag: the ETag of the remote fetch (None for the bundled backup) """ loaded_at: Final = _cost_map_source_info.loaded_at @@ -451,7 +442,6 @@ def get_model_cost_map_source_info() -> CostMapSourceInfo: "is_env_forced": _cost_map_source_info.is_env_forced, "fallback_reason": _cost_map_source_info.fallback_reason, "loaded_at": loaded_at.isoformat() if loaded_at is not None else None, - "generated_at": _cost_map_source_info.generated_at, "source_revision": _cost_map_source_info.source_revision, "etag": _cost_map_source_info.etag, } @@ -518,21 +508,24 @@ def _expand_model_aliases(model_cost: dict) -> dict: def _finalize_model_cost_map(model_cost: dict) -> dict: - """Extract fallback generalizations and the provenance stamp out of the raw map, then expand aliases. + """Extract fallback generalizations out of the raw map, then expand aliases. The ``fallback_generalizations`` block is installed into the generalizations - module and the ``_metadata`` block into the source info; both are removed from - the map so neither is ever treated as a model entry. + module and removed from the map so it is never treated as a model entry. """ raw: Final = model_cost.pop(FALLBACK_GENERALIZATIONS_KEY, None) rules: Final = raw.get("rules") if isinstance(raw, dict) else None set_fallback_generalizations(rules) - metadata: Final = _parse_metadata(model_cost.pop(METADATA_KEY, None)) - _cost_map_source_info.generated_at = metadata.generated_at - _cost_map_source_info.source_revision = metadata.source_revision return _expand_model_aliases(model_cost) +def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded: + """Record which bytes this process now serves, then finalize the map they parsed into""" + _cost_map_source_info.source_revision = loaded.revision + _cost_map_source_info.etag = loaded.etag + return replace(loaded, model_cost_map=_finalize_model_cost_map(loaded.model_cost_map)) + + def get_model_cost_map( url: str, timeout: int = 5, @@ -561,12 +554,10 @@ def get_model_cost_map( _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True _cost_map_source_info.fallback_reason = None - _cost_map_source_info.etag = None - return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False - _cost_map_source_info.etag = None result: Final = _fetch_remote_model_cost_map_with_retry_sync( url=url, @@ -584,7 +575,7 @@ def get_model_cost_map( ) _cost_map_source_info.source = "local" _cost_map_source_info.fallback_reason = f"Remote fetch failed: {result.reason}" - return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map content: Final = result.model_cost_map # Validate using cached count (cheap int comparison, no file I/O) @@ -598,9 +589,8 @@ def get_model_cost_map( ) _cost_map_source_info.source = "local" _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" - return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map _cost_map_source_info.source = "remote" _cost_map_source_info.fallback_reason = None - _cost_map_source_info.etag = result.etag - return _finalize_model_cost_map(content) + return _finalize_loaded_model_cost_map(result).model_cost_map diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5edb3c0e9d8..b1ffc1583e4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1,8 +1,4 @@ { - "_metadata": { - "generated_at": "2026-09-07T23:38:47Z", - "source_revision": "cd681a573fd9f5b6f15a1355f46178e4e9d374d2" - }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4741e4cd9d3..818a1506754 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -17938,7 +17938,7 @@ async def get_model_cost_map_source( - is_env_forced: true if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage - fallback_reason: human-readable reason why remote failed (null on success) - loaded_at: when this pod last loaded the map - - generated_at, source_revision: the _metadata stamp inside the loaded file + - source_revision: git blob id of the loaded file, what git rev-parse : prints for it - etag: the ETag of the remote fetch (null for the bundled backup) - model_count: number of models in the currently loaded cost map """ diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5edb3c0e9d8..b1ffc1583e4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,8 +1,4 @@ { - "_metadata": { - "generated_at": "2026-09-07T23:38:47Z", - "source_revision": "cd681a573fd9f5b6f15a1355f46178e4e9d374d2" - }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index c40c2a67682..47a1934a703 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -1,27 +1,9 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "LiteLLM model_prices_and_context_window.json", - "description": "Schema for LiteLLM's model price and context window registry (https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Every top-level key except '_metadata', 'sample_spec', and 'fallback_generalizations' is a model id, optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. All costs are USD per unit. New optional fields are added regularly, so consumers should ignore unknown fields rather than reject them.", + "description": "Schema for LiteLLM's model price and context window registry (https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. All costs are USD per unit. New optional fields are added regularly, so consumers should ignore unknown fields rather than reject them.", "type": "object", "properties": { - "_metadata": { - "type": "object", - "description": "Provenance of this file: when an automated sync last regenerated it and the commit it ran against. Human edits leave it untouched; not a model entry.", - "properties": { - "generated_at": { - "type": "string", - "format": "date-time" - }, - "source_revision": { - "type": "string" - } - }, - "required": [ - "generated_at", - "source_revision" - ], - "additionalProperties": false - }, "sample_spec": { "type": "object", "description": "Documentation placeholder illustrating the entry shape; not a real model and not schema-conformant (several values are prose)." diff --git a/scripts/sync_together_ai_models.py b/scripts/sync_together_ai_models.py index e009f1a7ce6..12b128890f1 100644 --- a/scripts/sync_together_ai_models.py +++ b/scripts/sync_together_ai_models.py @@ -19,11 +19,9 @@ import argparse import json import os import re -import subprocess import sys from collections.abc import Mapping, Sequence from dataclasses import dataclass, field -from datetime import datetime, timezone from pathlib import Path from types import MappingProxyType from typing import Final @@ -35,7 +33,6 @@ MODELS_URL: Final = "https://api.together.ai/v1/models?serverless" DEPRECATIONS_URL: Final = "https://docs.together.ai/docs/deprecations.md" PROVIDER: Final = "together_ai" PREFIX: Final = "together_ai/" -METADATA_KEY: Final = "_metadata" SOURCE_URL: Final = "https://docs.together.ai/docs/serverless-models" COST_MAP_RELPATHS: Final = ( "model_prices_and_context_window.json", @@ -498,23 +495,6 @@ def _serialize(cost_map: CostMap) -> str: return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n" -def stamp_metadata(cost_map: CostMap, generated_at: str, source_revision: str) -> CostMap: - return {**cost_map, METADATA_KEY: {"generated_at": generated_at, "source_revision": source_revision}} - - -def _utc_now_iso() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def _source_revision(repo_root: Path) -> str: - from_env: Final = os.environ.get("GITHUB_SHA") - if from_env: - return from_env - return subprocess.run( - ("git", "rev-parse", "HEAD"), cwd=repo_root, check=True, capture_output=True, text=True - ).stdout.strip() - - def main(argv: Sequence[str]) -> int: parser: Final = argparse.ArgumentParser(description=__doc__) parser.add_argument("--write", action="store_true", help="apply the sync to the cost map files (default: dry run)") @@ -547,9 +527,8 @@ def main(argv: Sequence[str]) -> int: if args.pr_body_file is not None: args.pr_body_file.write_text(body) if args.write and outcome.has_changes: - stamped: Final = _serialize(stamp_metadata(outcome.cost_map, _utc_now_iso(), _source_revision(args.repo_root))) for relpath in COST_MAP_RELPATHS: - (args.repo_root / relpath).write_text(stamped) + (args.repo_root / relpath).write_text(_serialize(outcome.cost_map)) print(render_summary(outcome)) print() print(body) diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 62f72495491..d9fe6d2f979 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -17,11 +17,11 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) from litellm.litellm_core_utils.get_model_cost_map import ( FALLBACK_GENERALIZATIONS_KEY, - METADATA_KEY, GetModelCostMap, _count_model_entries, _finalize_model_cost_map, get_model_cost_map_provenance, + git_blob_id, ) @@ -33,18 +33,16 @@ def _load_root_cost_map() -> dict: return json.load(f) -def _load_bundled_stamp() -> dict: - path = os.path.join( - os.path.dirname(__file__), "../../../litellm/model_prices_and_context_window_backup.json" - ) - with open(path) as f: - return json.load(f)[METADATA_KEY] +def _bundled_blob_id() -> str: + path = os.path.join(os.path.dirname(__file__), "../../../litellm/model_prices_and_context_window_backup.json") + with open(path, "rb") as f: + return git_blob_id(f.read()) -_STAMP = { - "generated_at": "2026-09-07T00:00:00Z", - "source_revision": "0123456789abcdef0123456789abcdef01234567", -} +def test_git_blob_id_is_what_git_hash_object_prints(): + """An operator checks a reported revision with ``git hash-object`` or ``git rev-parse :``, + so the id must be git's blob sha1 of the exact bytes, not a plain sha1 or a hash of the parsed JSON.""" + assert git_blob_id(b'{"gpt-5.4-mini": {"mode": "chat"}}\n') == "18b9a8381e13a3b38a2128f184f631f95829e987" def _make_models(n: int) -> dict: @@ -57,7 +55,6 @@ def test_count_model_entries_excludes_reserved_keys(): m = _make_models(3) m["sample_spec"] = {"foo": "bar"} m[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []} - m[METADATA_KEY] = dict(_STAMP) assert _count_model_entries(m) == 3 @@ -143,39 +140,6 @@ def test_finalize_with_no_block_clears_rules(): set_fallback_generalizations(previous) -def test_finalize_pops_metadata_and_records_provenance(): - finalized = _finalize_model_cost_map({**_make_models(2), METADATA_KEY: dict(_STAMP)}) - - assert METADATA_KEY not in finalized - assert len(finalized) == 2 - provenance = get_model_cost_map_provenance() - assert provenance["generated_at"] == _STAMP["generated_at"] - assert provenance["source_revision"] == _STAMP["source_revision"] - - -def test_finalize_without_metadata_clears_the_previous_stamp(): - _finalize_model_cost_map({**_make_models(2), METADATA_KEY: dict(_STAMP)}) - - _finalize_model_cost_map(_make_models(2)) - - provenance = get_model_cost_map_provenance() - assert provenance["generated_at"] is None - assert provenance["source_revision"] is None - - -@pytest.mark.parametrize( - "raw", - ["2026-09-07T00:00:00Z", {"generated_at": 42}, ["2026-09-07T00:00:00Z"]], - ids=["string", "wrong_field_type", "list"], -) -def test_finalize_tolerates_a_malformed_metadata_block(raw): - finalized = _finalize_model_cost_map({**_make_models(2), METADATA_KEY: raw}) - - assert METADATA_KEY not in finalized - assert len(finalized) == 2 - assert get_model_cost_map_provenance()["generated_at"] is None - - def test_shipped_backup_carries_the_claude_routing_rules(): """The bundled backup must ship the Claude routing rules so a fresh install (or an offline fallback) routes unknown Claude models without code changes. @@ -390,10 +354,6 @@ def _real_map_bytes() -> bytes: return json.dumps(_load_root_cost_map()).encode() -def _stamped_map_bytes(stamp: dict) -> bytes: - return json.dumps({**_load_root_cost_map(), METADATA_KEY: stamp}).encode() - - class _SleepRecorder: """Injected in place of asyncio.sleep so tests assert waits without real delay.""" @@ -555,40 +515,51 @@ async def test_refetch_respects_local_env_override(monkeypatch): @pytest.mark.asyncio -async def test_refetch_records_the_file_stamp_and_the_fetch_etag(): - """A reload reports which revision of the map it swapped in: the ``_metadata`` stamp the file - carries plus the ETag the fetch returned, with the stamp itself kept out of the model map.""" - client, _ = _mock_client( - [httpx.Response(200, headers={"ETag": 'W/"abc123"'}, content=_stamped_map_bytes(_STAMP))] - ) +async def test_refetch_records_the_blob_id_of_the_bytes_served_and_the_fetch_etag(): + """A reload reports which revision of the map it swapped in: the git blob id of the exact bytes the + fetch returned, so ``git rev-parse :model_prices_and_context_window.json`` can confirm it, + plus the ETag the fetch returned.""" + body = _real_map_bytes() + client, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"abc123"'}, content=body)]) result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloaded) + assert result.revision == git_blob_id(body) assert result.etag == 'W/"abc123"' - assert METADATA_KEY not in result.model_cost_map - assert get_model_cost_map_provenance() == { - "generated_at": _STAMP["generated_at"], - "source_revision": _STAMP["source_revision"], - "etag": 'W/"abc123"', - } + assert get_model_cost_map_provenance() == {"source_revision": git_blob_id(body), "etag": 'W/"abc123"'} @pytest.mark.asyncio -async def test_refetch_local_override_reports_the_bundled_stamp_without_an_etag(monkeypatch): - """Forcing the bundled backup after a remote reload must drop the remote ETag, since the map - served is no longer the one that ETag identifies.""" - remote, _ = _mock_client( - [httpx.Response(200, headers={"ETag": 'W/"remote"'}, content=_stamped_map_bytes(_STAMP))] +async def test_refetch_revision_follows_the_bytes_not_the_url(): + """Two fetches of the same URL that return different bytes report different revisions.""" + edited = json.loads(_real_map_bytes()) + edited["gpt-5.4-mini"]["input_cost_per_token"] = 0.5 + client, _ = _mock_client( + [httpx.Response(200, content=_real_map_bytes()), httpx.Response(200, content=json.dumps(edited).encode())] ) + + first = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) + second = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) + + assert isinstance(first, ModelCostMapReloaded) and isinstance(second, ModelCostMapReloaded) + assert first.revision != second.revision + assert get_model_cost_map_provenance()["source_revision"] == second.revision + + +@pytest.mark.asyncio +async def test_refetch_local_override_reports_the_bundled_blob_id_without_an_etag(monkeypatch): + """Forcing the bundled backup after a remote reload must report the backup's own blob id and drop the + remote ETag, since the map served is no longer the one that ETag identifies.""" + remote, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"remote"'}, content=_real_map_bytes())]) await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=remote) monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0)) assert isinstance(result, ModelCostMapReloaded) - assert METADATA_KEY not in result.model_cost_map - assert get_model_cost_map_provenance() == {**_load_bundled_stamp(), "etag": None} + assert result.revision == _bundled_blob_id() + assert get_model_cost_map_provenance() == {"source_revision": _bundled_blob_id(), "etag": None} # --------------------------------------------------------------------------- @@ -633,7 +604,7 @@ def test_boot_load_retries_transient_failures_instead_of_falling_back(): source = get_model_cost_map_source_info() assert source["source"] == "remote" assert source["fallback_reason"] is None - assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY, METADATA_KEY} + assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} def test_boot_load_honors_retry_after_then_falls_back_after_max_attempts(): @@ -685,37 +656,31 @@ def test_boot_load_respects_local_env_override(monkeypatch): assert get_model_cost_map_source_info()["is_env_forced"] is True -def test_boot_load_records_the_file_stamp_and_the_fetch_etag(): - client, _ = _mock_client( - [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_stamped_map_bytes(_STAMP))], - client_cls=httpx.Client, - ) +def test_boot_load_records_the_blob_id_of_the_bytes_served_and_the_fetch_etag(): + body = _real_map_bytes() + client, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=body)], client_cls=httpx.Client) - cost_map = get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=client) + get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=client) - assert METADATA_KEY not in cost_map source = get_model_cost_map_source_info() assert source["source"] == "remote" assert source["etag"] == 'W/"boot"' - assert source["generated_at"] == _STAMP["generated_at"] - assert source["source_revision"] == _STAMP["source_revision"] + assert source["source_revision"] == git_blob_id(body) assert source["loaded_at"] is not None -def test_boot_load_fallback_to_the_backup_drops_the_remote_etag(): - """A boot that lands on the bundled backup reports the backup's own stamp and no ETag, even +def test_boot_load_fallback_to_the_backup_reports_its_blob_id_and_drops_the_remote_etag(): + """A boot that lands on the bundled backup reports the backup's own blob id and no ETag, even when an earlier load in the same process had fetched the remote map.""" remote, _ = _mock_client( - [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_stamped_map_bytes(_STAMP))], - client_cls=httpx.Client, + [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_real_map_bytes())], client_cls=httpx.Client ) get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=remote) failing, _ = _mock_client([httpx.Response(404)], client_cls=httpx.Client) - cost_map = get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=failing) + get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=failing) - assert METADATA_KEY not in cost_map source = get_model_cost_map_source_info() assert source["source"] == "local" assert source["etag"] is None - assert {"generated_at": source["generated_at"], "source_revision": source["source_revision"]} == _load_bundled_stamp() + assert source["source_revision"] == _bundled_blob_id() diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py index fb3583a7dd2..0490993a314 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py @@ -22,7 +22,6 @@ from .conftest import VOLATILE_KEYS, normalize _VOLATILE = VOLATILE_KEYS | frozenset({"timestamp"}) _PROVENANCE = { - "generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef0123456789abcdef01234567", "etag": 'W/"cost-map-etag"', } @@ -108,22 +107,24 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): assert update_payload["reload_revision"] == {"increment": 1} -def test_reload_model_cost_map_surfaces_provenance_and_keeps_metadata_out_of_the_model_list( +def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every_status_surface( client, auth_as, monkeypatch, mock_prisma ): - """A real refetch through the reload route reports the file's stamp and the fetch ETag on every - status surface, while the ``_metadata`` block never shows up as a model anywhere.""" + """A real refetch through the reload route reports the git blob id of the exact bytes it fetched and + the fetch ETag on the reload response, the source route, and the schedule status alike.""" import httpx import litellm + from litellm.litellm_core_utils.get_model_cost_map import git_blob_id from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles _attach_litellm_config(mock_prisma) monkeypatch.setattr(ps, "prisma_client", mock_prisma) monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) - stamped = {**json.loads(_ROOT_COST_MAP.read_text()), "_metadata": {k: v for k, v in _PROVENANCE.items() if k != "etag"}} - served = httpx.Response(200, headers={"ETag": _PROVENANCE["etag"]}, content=json.dumps(stamped).encode()) + body = _ROOT_COST_MAP.read_bytes() + expected = {"source_revision": git_blob_id(body), "etag": _PROVENANCE["etag"]} + served = httpx.Response(200, headers={"ETag": _PROVENANCE["etag"]}, content=body) monkeypatch.setattr( "litellm.litellm_core_utils.get_model_cost_map._default_reload_client", lambda: httpx.AsyncClient(transport=httpx.MockTransport(lambda request: served)), @@ -144,18 +145,15 @@ def test_reload_model_cost_map_surfaces_provenance_and_keeps_metadata_out_of_the assert reload_response.status_code == 200 reload_body = reload_response.json() - assert {key: reload_body[key] for key in _PROVENANCE} == _PROVENANCE + assert {key: reload_body[key] for key in expected} == expected assert source_response.status_code == 200 source_body = source_response.json() - assert {key: source_body[key] for key in _PROVENANCE} == _PROVENANCE + assert {key: source_body[key] for key in expected} == expected assert source_body["source"] == "remote" assert status_response.status_code == 200 - assert {key: status_response.json()[key] for key in _PROVENANCE} == _PROVENANCE + assert {key: status_response.json()[key] for key in expected} == expected assert public_response.status_code == 200 - public_body = public_response.json() - assert "_metadata" not in public_body - assert "_metadata" not in litellm.model_cost - assert "gpt-4o" in public_body + assert "gpt-4o" in public_response.json() assert reload_body["models_count"] == len(litellm.model_cost) diff --git a/tests/test_litellm/test_auto_update_price_and_context_window_file.py b/tests/test_litellm/test_auto_update_price_and_context_window_file.py deleted file mode 100644 index d3cda09cd96..00000000000 --- a/tests/test_litellm/test_auto_update_price_and_context_window_file.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Tests for .github/scripts/auto_update_price_and_context_window_file.py.""" - -import importlib.util -import json -import re -import sys -from pathlib import Path -from typing import Final - -_REPO_ROOT: Final = Path(__file__).resolve().parents[2] -_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "auto_update_price_and_context_window_file.py" -_spec: Final = importlib.util.spec_from_file_location("auto_update_price_and_context_window_file", _MODULE_PATH) -script: Final = importlib.util.module_from_spec(_spec) -sys.modules[_spec.name] = script -_spec.loader.exec_module(script) - -_LOCAL_FILE: Final = "model_prices_and_context_window.json" -_GENERATED_AT: Final = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z") - - -def _openrouter_row(model_id: str) -> dict: - return {"id": model_id, "context_length": 8192, "pricing": {"prompt": "0.000001", "completion": "0.000002"}} - - -def _serve(openrouter_rows: list) -> object: - async def fetch_data(url: str) -> list: - return openrouter_rows if "openrouter" in url else [] - - return fetch_data - - -def _read_local(tmp_path: Path) -> dict: - return json.loads((tmp_path / _LOCAL_FILE).read_text()) - - -def test_main_stamps_provenance_only_when_the_sync_changed_the_file(tmp_path: Path, monkeypatch) -> None: - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("GITHUB_SHA", "feedface") - monkeypatch.setattr(script, "fetch_data", _serve([_openrouter_row("acme/x")])) - (tmp_path / _LOCAL_FILE).write_text(json.dumps({"sample_spec": {"input_cost_per_token": "USD"}}, indent=4) + "\n") - - script.main() - - written = _read_local(tmp_path) - assert written["openrouter/acme/x"]["litellm_provider"] == "openrouter" - assert written["_metadata"]["source_revision"] == "feedface" - assert _GENERATED_AT.fullmatch(written["_metadata"]["generated_at"]) - - sentinel = {**written, "_metadata": {**written["_metadata"], "generated_at": "2000-01-01T00:00:00Z"}} - (tmp_path / _LOCAL_FILE).write_text(json.dumps(sentinel, indent=4) + "\n") - - script.main() - - assert _read_local(tmp_path) == sentinel diff --git a/tests/test_litellm/test_cost_map_guard.py b/tests/test_litellm/test_cost_map_guard.py index 1a60cf81164..1b4330ed62c 100644 --- a/tests/test_litellm/test_cost_map_guard.py +++ b/tests/test_litellm/test_cost_map_guard.py @@ -141,26 +141,6 @@ def test_bot_may_not_change_special_root_keys() -> None: assert _failures(head) == ("bot PRs may not change fallback_generalizations",) -STAMP: Final = {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef0123456789abcdef01234567"} - - -def test_bot_may_stamp_and_restamp_metadata() -> None: - stamped = _snapshot({**BASE_MAP, "_metadata": STAMP}) - assert _failures(stamped) == () - assert _failures(stamped, bot=False) == () - - restamped = _snapshot( - { - **BASE_MAP, - "_metadata": {**STAMP, "generated_at": "2026-09-14T00:00:00Z"}, - "fallback_generalizations": {"rules": []}, - } - ) - assert guard.guard_failures(stamped, restamped, MAP_FILES, True) == ( - "bot PRs may not change fallback_generalizations", - ) - - def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str: text = _serialize(cost_map) (repo / guard.COST_MAP_PATH).write_text(text) diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index 3517f5840e8..c2c22c25998 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -98,25 +98,6 @@ def test_schema_rejects_malformed_entries(committed_schema: dict, entry: dict): assert not validator.is_valid({"some-model": entry}) -@pytest.mark.parametrize( - "metadata", - [ - "2026-09-07T00:00:00Z", - {"generated_at": "2026-09-07T00:00:00Z"}, - {"source_revision": "0123456789abcdef"}, - {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef", "author": "bot"}, - ], - ids=["not_an_object", "missing_revision", "missing_generated_at", "unknown_field"], -) -def test_schema_rejects_a_malformed_metadata_block(committed_schema: dict, metadata: object): - assert not build_validator(committed_schema).is_valid({"_metadata": metadata}) - - -def test_schema_accepts_the_provenance_stamp_as_a_non_model_root_key(committed_schema: dict): - stamp = {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef0123456789abcdef01234567"} - assert build_validator(committed_schema).is_valid({"_metadata": stamp}) - - def test_schema_accepts_minimal_and_unknown_optional_fields(committed_schema: dict): validator = build_validator(committed_schema) assert validator.is_valid({"some-model": {"litellm_provider": "openai"}}) diff --git a/tests/test_litellm/test_sync_together_ai_models.py b/tests/test_litellm/test_sync_together_ai_models.py index f58f573c208..b8a85bcfbdc 100644 --- a/tests/test_litellm/test_sync_together_ai_models.py +++ b/tests/test_litellm/test_sync_together_ai_models.py @@ -1,6 +1,5 @@ import importlib.util import json -import re from pathlib import Path from types import MappingProxyType @@ -370,58 +369,6 @@ def test_sync_is_idempotent_over_the_repo_cost_map() -> None: assert second.cost_map == first.cost_map -def test_stamp_metadata_adds_the_provenance_block_without_touching_models() -> None: - cost_map = {"sample_spec": {"input_cost_per_token": "USD"}, "together_ai/acme/x": {"mode": "chat"}} - - stamped = sync.stamp_metadata(cost_map, "2026-09-07T00:00:00Z", "feedface") - - assert stamped["_metadata"] == {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "feedface"} - assert {key: value for key, value in stamped.items() if key != "_metadata"} == cost_map - assert "_metadata" not in cost_map - - -def _write_registry(repo_root: Path, cost_map: dict) -> None: - for relpath in sync.COST_MAP_RELPATHS: - target = repo_root / relpath - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(json.dumps(cost_map, indent=4) + "\n") - - -def _read_registries(repo_root: Path) -> tuple[dict, ...]: - return tuple(json.loads((repo_root / relpath).read_text()) for relpath in sync.COST_MAP_RELPATHS) - - -def test_write_stamps_provenance_into_both_files_only_when_the_sync_changed_them(tmp_path: Path, monkeypatch) -> None: - monkeypatch.setenv("GITHUB_SHA", "feedface") - cost_map = json.loads((ROOT / "model_prices_and_context_window.json").read_text()) - dropped = next(f"together_ai/{model.id}" for model in RECORDED_CATALOG if f"together_ai/{model.id}" in cost_map) - _write_registry(tmp_path, {key: value for key, value in cost_map.items() if key not in {dropped, "_metadata"}}) - argv = ( - "--write", - "--models-json", - str(FIXTURES / "models_serverless.json"), - "--deprecations-md", - str(FIXTURES / "deprecations.md"), - "--repo-root", - str(tmp_path), - ) - - assert sync.main(argv) == 0 - - written = _read_registries(tmp_path) - assert written[0] == written[1] - assert dropped in written[0] - assert written[0]["_metadata"]["source_revision"] == "feedface" - assert re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", written[0]["_metadata"]["generated_at"]) - - sentinel = {**written[0], "_metadata": {**written[0]["_metadata"], "generated_at": "2000-01-01T00:00:00Z"}} - _write_registry(tmp_path, sentinel) - - assert sync.main(argv) == 0 - - assert _read_registries(tmp_path) == (sentinel, sentinel) - - def test_pr_body_lists_every_section_and_the_skipped_types() -> None: outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) body = sync.render_pr_body(outcome) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index f6c9a4537a1..8a56a84ade7 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -21,7 +21,6 @@ from litellm._logging import ( verbose_logger, ) from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.get_model_cost_map import RESERVED_TOP_LEVEL_KEYS from litellm.proxy.utils import is_valid_api_key from litellm.types.utils import ( CallTypes, @@ -1219,12 +1218,15 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): with open(prod_json, "r") as model_prices_file: actual_json = json.load(model_prices_file) assert isinstance(actual_json, dict) - model_entries: Final = { - key: value for key, value in actual_json.items() if key not in RESERVED_TOP_LEVEL_KEYS - } + actual_json.pop( + "sample_spec", None + ) # remove the sample, whose schema is inconsistent with the real data + actual_json.pop( + "fallback_generalizations", None + ) # reserved meta key, not a model entry # Validate schema - validate(model_entries, INTENDED_SCHEMA) + validate(actual_json, INTENDED_SCHEMA) # Validate cost values # Define exceptions for models that are allowed to have costs > 1 @@ -1235,7 +1237,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "runwayml/seedance2", # 4K output is 150 credits/second = $1.50/second ] - is_valid, violations = validate_model_cost_values(model_entries, exceptions) + is_valid, violations = validate_model_cost_values(actual_json, exceptions) if not is_valid: error_message = "Cost validation failed:\n" + "\n".join(violations) @@ -1266,7 +1268,8 @@ def test_max_tokens_consistency(): inconsistencies = [] for model_name, config in models.items(): - if model_name in RESERVED_TOP_LEVEL_KEYS: + # Skip the sample_spec + if model_name == "sample_spec": continue # Check if both max_tokens and max_output_tokens exist diff --git a/ui/litellm-dashboard/src/components/price_data_reload.test.tsx b/ui/litellm-dashboard/src/components/price_data_reload.test.tsx index fde69675b72..101612993b0 100644 --- a/ui/litellm-dashboard/src/components/price_data_reload.test.tsx +++ b/ui/litellm-dashboard/src/components/price_data_reload.test.tsx @@ -33,15 +33,13 @@ const remoteSource = { is_env_forced: false, fallback_reason: null, loaded_at: null, - generated_at: null, source_revision: null, etag: null, model_count: 1234, }; const provenance = { loaded_at: "2026-09-07T10:00:00Z", - generated_at: "2026-09-06T23:38:47Z", - source_revision: "cd681a573fd9f5b6f15a1355f46178e4e9d374d2", + source_revision: "4273ec544726bf255ea920533e209e6022653bb4", etag: 'W/"eb8e9a53f4cc284b"', }; @@ -66,24 +64,22 @@ describe("PriceDataReload", () => { render(); expect(await screen.findByText("Source revision:")).toBeInTheDocument(); - expect(screen.getByText("cd681a573fd9")).toBeInTheDocument(); + expect(screen.getByText("4273ec544726")).toBeInTheDocument(); expect(screen.getByText("ETag:")).toBeInTheDocument(); expect(screen.getByText('W/"eb8e9a53f4cc284b"')).toBeInTheDocument(); - expect(screen.getByText("Generated at:")).toBeInTheDocument(); - expect(screen.getByText(new Date(provenance.generated_at).toLocaleString())).toBeInTheDocument(); expect(screen.getByText("Loaded at:")).toBeInTheDocument(); expect(screen.getByText(new Date(provenance.loaded_at).toLocaleString())).toBeInTheDocument(); }); - it("shows a malformed generated_at stamp as-is instead of Invalid Date", async () => { + it("shows a malformed loaded_at as-is instead of Invalid Date", async () => { vi.mocked(getModelCostMapSource).mockResolvedValue({ ...remoteSource, ...provenance, - generated_at: "yesterday-ish", + loaded_at: "yesterday-ish", } as never); render(); - expect(await screen.findByText("Generated at:")).toBeInTheDocument(); + expect(await screen.findByText("Loaded at:")).toBeInTheDocument(); expect(screen.getByText("yesterday-ish")).toBeInTheDocument(); expect(screen.queryByText("Invalid Date")).not.toBeInTheDocument(); }); @@ -92,7 +88,6 @@ describe("PriceDataReload", () => { render(); expect(await screen.findByText("Pricing Data Source")).toBeInTheDocument(); - expect(screen.queryByText("Generated at:")).not.toBeInTheDocument(); expect(screen.queryByText("Source revision:")).not.toBeInTheDocument(); expect(screen.queryByText("ETag:")).not.toBeInTheDocument(); expect(screen.queryByText("Loaded at:")).not.toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/price_data_reload.tsx b/ui/litellm-dashboard/src/components/price_data_reload.tsx index c152916f271..e5977a1b6e3 100644 --- a/ui/litellm-dashboard/src/components/price_data_reload.tsx +++ b/ui/litellm-dashboard/src/components/price_data_reload.tsx @@ -50,7 +50,6 @@ interface CostMapSourceInfo { is_env_forced: boolean; fallback_reason: string | null; loaded_at: string | null; - generated_at: string | null; source_revision: string | null; etag: string | null; model_count: number; @@ -105,13 +104,6 @@ const formatDateTime = (dateTimeString: string | null) => { const CostMapProvenanceRows: React.FC<{ sourceInfo: CostMapSourceInfo }> = ({ sourceInfo }) => ( <> - {sourceInfo.generated_at && ( -
- Generated at: - {formatDateTime(sourceInfo.generated_at)} -
- )} - {sourceInfo.source_revision && (
Source revision: diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7b9b24c9627..fc73d8264ef 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8685,7 +8685,7 @@ export interface paths { * - is_env_forced: true if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage * - fallback_reason: human-readable reason why remote failed (null on success) * - loaded_at: when this pod last loaded the map - * - generated_at, source_revision: the _metadata stamp inside the loaded file + * - source_revision: git blob id of the loaded file, what git rev-parse : prints for it * - etag: the ETag of the remote fetch (null for the bundled backup) * - model_count: number of models in the currently loaded cost map */