From 0710231acc349f1cb8229fb5d691678dc2402e80 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 16:57:09 -0700 Subject: [PATCH 01/14] feat(cost_map): stamp and surface generated_at and source revision provenance The cost map JSON now carries a top-level `_metadata` block with `generated_at` and `source_revision`, written by the two bot writers only when model data changed. The loader pops it before the map becomes `litellm.model_cost`, records it next to the fetch ETag, and `/reload/model_cost_map`, `/model/cost_map/source`, and the reload schedule status return it. The Price Data Reload card shows the stamp, the ETag, and when the pod loaded the map. The schema and the cost map guard treat `_metadata` as a non-model root key --- ...to_update_price_and_context_window_file.py | 27 +++- ci_cd/cost_map_guard.py | 7 +- ci_cd/generate_model_prices_schema.py | 19 ++- .../litellm_core_utils/get_model_cost_map.py | 82 ++++++++++- ...odel_prices_and_context_window_backup.json | 4 + litellm/proxy/proxy_server.py | 15 +- model_prices_and_context_window.json | 4 + model_prices_and_context_window.schema.json | 20 ++- scripts/sync_together_ai_models.py | 23 +++- .../test_get_model_cost_map.py | 129 +++++++++++++++++- .../test_routes_model_cost_map.py | 83 ++++++++++- ...to_update_price_and_context_window_file.py | 54 ++++++++ tests/test_litellm/test_cost_map_guard.py | 20 +++ .../test_litellm/test_model_prices_schema.py | 19 +++ .../test_sync_together_ai_models.py | 53 +++++++ .../src/components/price_data_reload.test.tsx | 34 +++++ .../src/components/price_data_reload.tsx | 68 +++++++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 3 + 18 files changed, 636 insertions(+), 28 deletions(-) create mode 100644 tests/test_litellm/test_auto_update_price_and_context_window_file.py diff --git a/.github/scripts/auto_update_price_and_context_window_file.py b/.github/scripts/auto_update_price_and_context_window_file.py index 461d8d347d9..a7a3194f262 100644 --- a/.github/scripts/auto_update_price_and_context_window_file.py +++ b/.github/scripts/auto_update_price_and_context_window_file.py @@ -1,6 +1,9 @@ import asyncio import aiohttp import json +import os +import subprocess +from datetime import datetime, timezone # Asynchronously fetch data from a given URL async def fetch_data(url): @@ -31,13 +34,28 @@ def sync_local_data_with_remote(local_data, remote_data): for key in (set(remote_data) - set(local_data)): local_data[key] = remote_data[key] +def utc_now_iso(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def source_revision(): + from_env = os.environ.get("GITHUB_SHA") + if from_env: + return from_env + return subprocess.run(["git", "rev-parse", "HEAD"], check=True, capture_output=True, text=True).stdout.strip() + + +def stamp_metadata(data, generated_at, revision): + return {**data, "_metadata": {"generated_at": generated_at, "source_revision": revision}} + + # Write data to the json file def write_to_file(file_path, data): try: # Open the file in write mode with open(file_path, "w") as file: # Dump the data as JSON into the file - json.dump(data, file, indent=4) + file.write(json.dumps(data, indent=4) + "\n") print("Values updated successfully.") except Exception as e: # Print an error message if writing to file fails @@ -149,8 +167,13 @@ def main(): # If both local and openrouter data are available, synchronize and save if local_data and all_remote_data: + before = json.dumps(local_data, sort_keys=True) sync_local_data_with_remote(local_data, all_remote_data) - write_to_file(local_file_path, local_data) + changed = json.dumps(local_data, sort_keys=True) != before + write_to_file( + local_file_path, + stamp_metadata(local_data, utc_now_iso(), source_revision()) if changed else local_data, + ) else: print("Failed to fetch model data from either local file or URL.") diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 50aa40ba220..351c06c74eb 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -2,7 +2,8 @@ Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named -litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models. +litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models, plus +restamp the _metadata provenance block. """ from __future__ import annotations @@ -15,7 +16,7 @@ from collections.abc import Sequence from dataclasses import dataclass from typing import Final -from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors +from generate_model_prices_schema import BOT_LOCKED_ROOT_KEYS, build_schema, render, validation_errors COST_MAP_PATH: Final = "model_prices_and_context_window.json" BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" @@ -102,7 +103,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str *(f"bot PRs may not remove fields: {ref}" for ref in removed_fields), *( f"bot PRs may not change {key}" - for key in sorted(SPECIAL_ROOT_KEYS) + for key in sorted(BOT_LOCKED_ROOT_KEYS) if base_map.get(key) != head_map.get(key) ), ) diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index ab29b70bdd4..557afa50128 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -11,7 +11,9 @@ REPO_ROOT = Path(__file__).parent.parent PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json" -SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"}) +METADATA_KEY = "_metadata" +SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations", METADATA_KEY}) +BOT_LOCKED_ROOT_KEYS = SPECIAL_ROOT_KEYS - {METADATA_KEY} JsonSchema = dict @@ -271,13 +273,26 @@ def build_schema(prices: dict) -> JsonSchema: "description": ( "Schema for LiteLLM's model price and context window registry " "(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). " - "Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, " + "Every top-level key except '_metadata', 'sample_spec', and 'fallback_generalizations' is a model id, " "optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. " "All costs are USD per unit. New optional fields are added regularly, so consumers should " "ignore unknown fields rather than reject them." ), "type": "object", "properties": { + METADATA_KEY: { + "type": "object", + "description": ( + "Provenance of this file: when an automated sync last regenerated it and the commit it " + "ran against. Human edits leave it untouched; not a model entry." + ), + "properties": { + "generated_at": {"type": "string", "format": "date-time"}, + "source_revision": STRING, + }, + "required": ["generated_at", "source_revision"], + "additionalProperties": False, + }, "sample_spec": { "type": "object", "description": ( diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index ba8738c8de0..a538e7cb330 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -20,6 +20,8 @@ from importlib.resources import files from typing import Final, Protocol import httpx +from pydantic import BaseModel, ConfigDict, ValidationError +from typing_extensions import ReadOnly, TypedDict from litellm import verbose_logger from litellm.constants import ( @@ -31,10 +33,11 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" +METADATA_KEY: Final = "_metadata" # Reserved top-level keys that are not model entries. They must be excluded # from the model-count integrity check so a real upstream shrink can't be masked. -RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY}) +RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY, METADATA_KEY}) def _count_model_entries(model_cost: dict) -> int: @@ -166,6 +169,7 @@ MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0 @dataclass(frozen=True, slots=True) class ModelCostMapReloaded: model_cost_map: dict # mutable-ok: adopted as litellm.model_cost, whose consumer contract is a plain mutable dict + etag: str | None = None @dataclass(frozen=True, slots=True) @@ -254,7 +258,7 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}") if not isinstance(parsed, dict): return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}") - return ModelCostMapReloaded(model_cost_map=parsed) + return ModelCostMapReloaded(model_cost_map=parsed, etag=response.headers.get("etag")) def _next_retry_wait( @@ -328,10 +332,12 @@ async def refetch_model_cost_map( map they already have. """ if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + _cost_map_source_info.loaded_at = datetime.now(timezone.utc) _cost_map_source_info.source = "local" _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True _cost_map_source_info.fallback_reason = None + _cost_map_source_info.etag = None return ModelCostMapReloaded( model_cost_map=_finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) ) @@ -355,11 +361,13 @@ async def refetch_model_cost_map( backup_model_count=GetModelCostMap._get_backup_model_count(), ): return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} failed integrity validation") + _cost_map_source_info.loaded_at = datetime.now(timezone.utc) _cost_map_source_info.source = "remote" _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False _cost_map_source_info.fallback_reason = None - return ModelCostMapReloaded(model_cost_map=_finalize_model_cost_map(result.model_cost_map)) + _cost_map_source_info.etag = result.etag + return ModelCostMapReloaded(model_cost_map=_finalize_model_cost_map(result.model_cost_map), etag=result.etag) class ModelCostMapSourceInfo: @@ -370,13 +378,60 @@ class ModelCostMapSourceInfo: is_env_forced: bool = False fallback_reason: str | None = None loaded_at: "datetime | None" = None + generated_at: str | None = None + source_revision: str | None = None + etag: str | None = None # Module-level singleton tracking the source of the current cost map _cost_map_source_info: Final = ModelCostMapSourceInfo() -def get_model_cost_map_source_info() -> dict: +class CostMapMetadata(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + generated_at: str | None = None + source_revision: str | None = None + + +_EMPTY_METADATA: Final = CostMapMetadata() + + +def _parse_metadata(raw: object) -> CostMapMetadata: + if raw is None: + return _EMPTY_METADATA + try: + return CostMapMetadata.model_validate(raw) + except ValidationError as error: + verbose_logger.warning("LiteLLM: ignoring a malformed %s block in the model cost map: %s", METADATA_KEY, error) + return _EMPTY_METADATA + + +class CostMapProvenance(TypedDict): + generated_at: ReadOnly[str | None] + source_revision: ReadOnly[str | None] + etag: ReadOnly[str | None] + + +class CostMapSourceInfo(CostMapProvenance): + source: ReadOnly[str] + url: ReadOnly[str | None] + is_env_forced: ReadOnly[bool] + fallback_reason: ReadOnly[str | None] + loaded_at: ReadOnly[str | None] + + +def get_model_cost_map_provenance() -> CostMapProvenance: + """Which revision of the cost map this process serves: the ``_metadata`` stamp the file + carries plus the ETag the remote fetch returned (None for the bundled backup)""" + return { + "generated_at": _cost_map_source_info.generated_at, + "source_revision": _cost_map_source_info.source_revision, + "etag": _cost_map_source_info.etag, + } + + +def get_model_cost_map_source_info() -> CostMapSourceInfo: """ Return metadata about where the current model cost map was loaded from. @@ -385,12 +440,20 @@ def get_model_cost_map_source_info() -> dict: - url: the remote URL attempted (or None for local-only) - is_env_forced: True if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage - fallback_reason: human-readable reason if remote failed and local was used + - loaded_at: ISO 8601 time this process last loaded the map + - generated_at, source_revision: the ``_metadata`` stamp inside the loaded file + - etag: the ETag of the remote fetch (None for the bundled backup) """ + loaded_at: Final = _cost_map_source_info.loaded_at return { "source": _cost_map_source_info.source, "url": _cost_map_source_info.url, "is_env_forced": _cost_map_source_info.is_env_forced, "fallback_reason": _cost_map_source_info.fallback_reason, + "loaded_at": loaded_at.isoformat() if loaded_at is not None else None, + "generated_at": _cost_map_source_info.generated_at, + "source_revision": _cost_map_source_info.source_revision, + "etag": _cost_map_source_info.etag, } @@ -455,14 +518,18 @@ def _expand_model_aliases(model_cost: dict) -> dict: def _finalize_model_cost_map(model_cost: dict) -> dict: - """Extract fallback generalizations out of the raw map, then expand aliases. + """Extract fallback generalizations and the provenance stamp out of the raw map, then expand aliases. The ``fallback_generalizations`` block is installed into the generalizations - module and removed from the map so it is never treated as a model entry. + module and the ``_metadata`` block into the source info; both are removed from + the map so neither is ever treated as a model entry. """ raw: Final = model_cost.pop(FALLBACK_GENERALIZATIONS_KEY, None) rules: Final = raw.get("rules") if isinstance(raw, dict) else None set_fallback_generalizations(rules) + metadata: Final = _parse_metadata(model_cost.pop(METADATA_KEY, None)) + _cost_map_source_info.generated_at = metadata.generated_at + _cost_map_source_info.source_revision = metadata.source_revision return _expand_model_aliases(model_cost) @@ -494,10 +561,12 @@ def get_model_cost_map( _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True _cost_map_source_info.fallback_reason = None + _cost_map_source_info.etag = None return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False + _cost_map_source_info.etag = None result: Final = _fetch_remote_model_cost_map_with_retry_sync( url=url, @@ -533,4 +602,5 @@ def get_model_cost_map( _cost_map_source_info.source = "remote" _cost_map_source_info.fallback_reason = None + _cost_map_source_info.etag = result.etag return _finalize_model_cost_map(content) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b1ffc1583e4..5edb3c0e9d8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1,4 +1,8 @@ { + "_metadata": { + "generated_at": "2026-09-07T23:38:47Z", + "source_revision": "cd681a573fd9f5b6f15a1355f46178e4e9d374d2" + }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0915b8dd1b9..4741e4cd9d3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -17749,6 +17749,7 @@ async def reload_model_cost_map( # Immediately reload the model cost map in the current pod from litellm.litellm_core_utils.get_model_cost_map import ( ModelCostMapReloadUnavailable, + get_model_cost_map_provenance, refetch_model_cost_map, ) @@ -17762,6 +17763,7 @@ async def reload_model_cost_map( models_count = _swap_in_model_cost_map(reload_result.model_cost_map) current_time = utc_now() proxy_config.model_cost_map_loaded_at = current_time + provenance: Final = get_model_cost_map_provenance() # Publish a new revision so every other pod reloads on its next poll; this pod has # already served it, so adopt it here rather than reloading again a tick later @@ -17776,6 +17778,7 @@ async def reload_model_cost_map( "status": "success", "models_count": models_count, "timestamp": current_time.isoformat(), + **provenance, } except HTTPException: raise @@ -17896,12 +17899,17 @@ async def get_model_cost_map_reload_status( try: global prisma_client + from litellm.litellm_core_utils.get_model_cost_map import ( + get_model_cost_map_provenance, + ) + provenance: Final = get_model_cost_map_provenance() if prisma_client is None: verbose_proxy_logger.info("No database connection, returning not scheduled") - return reload_schedule_status(None) + return {**reload_schedule_status(None), **provenance} - return reload_schedule_status(await read_reload_schedule(prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME)) + schedule: Final = await read_reload_schedule(prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME) + return {**reload_schedule_status(schedule), **provenance} except Exception as e: verbose_proxy_logger.exception("Failed to get model cost map reload status: %s", e) raise HTTPException( @@ -17929,6 +17937,9 @@ async def get_model_cost_map_source( - url: the remote URL that was attempted (null when env-forced local) - is_env_forced: true if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage - fallback_reason: human-readable reason why remote failed (null on success) + - loaded_at: when this pod last loaded the map + - generated_at, source_revision: the _metadata stamp inside the loaded file + - etag: the ETag of the remote fetch (null for the bundled backup) - model_count: number of models in the currently loaded cost map """ # Read-only source info — admin viewers can read. diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b1ffc1583e4..5edb3c0e9d8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,4 +1,8 @@ { + "_metadata": { + "generated_at": "2026-09-07T23:38:47Z", + "source_revision": "cd681a573fd9f5b6f15a1355f46178e4e9d374d2" + }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 47a1934a703..c40c2a67682 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -1,9 +1,27 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "LiteLLM model_prices_and_context_window.json", - "description": "Schema for LiteLLM's model price and context window registry (https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. All costs are USD per unit. New optional fields are added regularly, so consumers should ignore unknown fields rather than reject them.", + "description": "Schema for LiteLLM's model price and context window registry (https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Every top-level key except '_metadata', 'sample_spec', and 'fallback_generalizations' is a model id, optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. All costs are USD per unit. New optional fields are added regularly, so consumers should ignore unknown fields rather than reject them.", "type": "object", "properties": { + "_metadata": { + "type": "object", + "description": "Provenance of this file: when an automated sync last regenerated it and the commit it ran against. Human edits leave it untouched; not a model entry.", + "properties": { + "generated_at": { + "type": "string", + "format": "date-time" + }, + "source_revision": { + "type": "string" + } + }, + "required": [ + "generated_at", + "source_revision" + ], + "additionalProperties": false + }, "sample_spec": { "type": "object", "description": "Documentation placeholder illustrating the entry shape; not a real model and not schema-conformant (several values are prose)." diff --git a/scripts/sync_together_ai_models.py b/scripts/sync_together_ai_models.py index 12b128890f1..e009f1a7ce6 100644 --- a/scripts/sync_together_ai_models.py +++ b/scripts/sync_together_ai_models.py @@ -19,9 +19,11 @@ import argparse import json import os import re +import subprocess import sys from collections.abc import Mapping, Sequence from dataclasses import dataclass, field +from datetime import datetime, timezone from pathlib import Path from types import MappingProxyType from typing import Final @@ -33,6 +35,7 @@ MODELS_URL: Final = "https://api.together.ai/v1/models?serverless" DEPRECATIONS_URL: Final = "https://docs.together.ai/docs/deprecations.md" PROVIDER: Final = "together_ai" PREFIX: Final = "together_ai/" +METADATA_KEY: Final = "_metadata" SOURCE_URL: Final = "https://docs.together.ai/docs/serverless-models" COST_MAP_RELPATHS: Final = ( "model_prices_and_context_window.json", @@ -495,6 +498,23 @@ def _serialize(cost_map: CostMap) -> str: return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n" +def stamp_metadata(cost_map: CostMap, generated_at: str, source_revision: str) -> CostMap: + return {**cost_map, METADATA_KEY: {"generated_at": generated_at, "source_revision": source_revision}} + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _source_revision(repo_root: Path) -> str: + from_env: Final = os.environ.get("GITHUB_SHA") + if from_env: + return from_env + return subprocess.run( + ("git", "rev-parse", "HEAD"), cwd=repo_root, check=True, capture_output=True, text=True + ).stdout.strip() + + def main(argv: Sequence[str]) -> int: parser: Final = argparse.ArgumentParser(description=__doc__) parser.add_argument("--write", action="store_true", help="apply the sync to the cost map files (default: dry run)") @@ -527,8 +547,9 @@ def main(argv: Sequence[str]) -> int: if args.pr_body_file is not None: args.pr_body_file.write_text(body) if args.write and outcome.has_changes: + stamped: Final = _serialize(stamp_metadata(outcome.cost_map, _utc_now_iso(), _source_revision(args.repo_root))) for relpath in COST_MAP_RELPATHS: - (args.repo_root / relpath).write_text(_serialize(outcome.cost_map)) + (args.repo_root / relpath).write_text(stamped) print(render_summary(outcome)) print() print(body) diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 18185126775..62f72495491 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -17,9 +17,11 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) from litellm.litellm_core_utils.get_model_cost_map import ( FALLBACK_GENERALIZATIONS_KEY, + METADATA_KEY, GetModelCostMap, _count_model_entries, _finalize_model_cost_map, + get_model_cost_map_provenance, ) @@ -31,6 +33,20 @@ def _load_root_cost_map() -> dict: return json.load(f) +def _load_bundled_stamp() -> dict: + path = os.path.join( + os.path.dirname(__file__), "../../../litellm/model_prices_and_context_window_backup.json" + ) + with open(path) as f: + return json.load(f)[METADATA_KEY] + + +_STAMP = { + "generated_at": "2026-09-07T00:00:00Z", + "source_revision": "0123456789abcdef0123456789abcdef01234567", +} + + def _make_models(n: int) -> dict: return { f"model-{i}": {"litellm_provider": "openai", "mode": "chat"} for i in range(n) @@ -41,6 +57,7 @@ def test_count_model_entries_excludes_reserved_keys(): m = _make_models(3) m["sample_spec"] = {"foo": "bar"} m[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []} + m[METADATA_KEY] = dict(_STAMP) assert _count_model_entries(m) == 3 @@ -126,6 +143,39 @@ def test_finalize_with_no_block_clears_rules(): set_fallback_generalizations(previous) +def test_finalize_pops_metadata_and_records_provenance(): + finalized = _finalize_model_cost_map({**_make_models(2), METADATA_KEY: dict(_STAMP)}) + + assert METADATA_KEY not in finalized + assert len(finalized) == 2 + provenance = get_model_cost_map_provenance() + assert provenance["generated_at"] == _STAMP["generated_at"] + assert provenance["source_revision"] == _STAMP["source_revision"] + + +def test_finalize_without_metadata_clears_the_previous_stamp(): + _finalize_model_cost_map({**_make_models(2), METADATA_KEY: dict(_STAMP)}) + + _finalize_model_cost_map(_make_models(2)) + + provenance = get_model_cost_map_provenance() + assert provenance["generated_at"] is None + assert provenance["source_revision"] is None + + +@pytest.mark.parametrize( + "raw", + ["2026-09-07T00:00:00Z", {"generated_at": 42}, ["2026-09-07T00:00:00Z"]], + ids=["string", "wrong_field_type", "list"], +) +def test_finalize_tolerates_a_malformed_metadata_block(raw): + finalized = _finalize_model_cost_map({**_make_models(2), METADATA_KEY: raw}) + + assert METADATA_KEY not in finalized + assert len(finalized) == 2 + assert get_model_cost_map_provenance()["generated_at"] is None + + def test_shipped_backup_carries_the_claude_routing_rules(): """The bundled backup must ship the Claude routing rules so a fresh install (or an offline fallback) routes unknown Claude models without code changes. @@ -340,6 +390,10 @@ def _real_map_bytes() -> bytes: return json.dumps(_load_root_cost_map()).encode() +def _stamped_map_bytes(stamp: dict) -> bytes: + return json.dumps({**_load_root_cost_map(), METADATA_KEY: stamp}).encode() + + class _SleepRecorder: """Injected in place of asyncio.sleep so tests assert waits without real delay.""" @@ -500,6 +554,43 @@ async def test_refetch_respects_local_env_override(monkeypatch): assert len(result.model_cost_map) > 100 +@pytest.mark.asyncio +async def test_refetch_records_the_file_stamp_and_the_fetch_etag(): + """A reload reports which revision of the map it swapped in: the ``_metadata`` stamp the file + carries plus the ETag the fetch returned, with the stamp itself kept out of the model map.""" + client, _ = _mock_client( + [httpx.Response(200, headers={"ETag": 'W/"abc123"'}, content=_stamped_map_bytes(_STAMP))] + ) + + result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) + + assert isinstance(result, ModelCostMapReloaded) + assert result.etag == 'W/"abc123"' + assert METADATA_KEY not in result.model_cost_map + assert get_model_cost_map_provenance() == { + "generated_at": _STAMP["generated_at"], + "source_revision": _STAMP["source_revision"], + "etag": 'W/"abc123"', + } + + +@pytest.mark.asyncio +async def test_refetch_local_override_reports_the_bundled_stamp_without_an_etag(monkeypatch): + """Forcing the bundled backup after a remote reload must drop the remote ETag, since the map + served is no longer the one that ETag identifies.""" + remote, _ = _mock_client( + [httpx.Response(200, headers={"ETag": 'W/"remote"'}, content=_stamped_map_bytes(_STAMP))] + ) + await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=remote) + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0)) + + assert isinstance(result, ModelCostMapReloaded) + assert METADATA_KEY not in result.model_cost_map + assert get_model_cost_map_provenance() == {**_load_bundled_stamp(), "etag": None} + + # --------------------------------------------------------------------------- # get_model_cost_map: the boot-time load retries transient failures like a reload does # --------------------------------------------------------------------------- @@ -542,7 +633,7 @@ def test_boot_load_retries_transient_failures_instead_of_falling_back(): source = get_model_cost_map_source_info() assert source["source"] == "remote" assert source["fallback_reason"] is None - assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} + assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY, METADATA_KEY} def test_boot_load_honors_retry_after_then_falls_back_after_max_attempts(): @@ -592,3 +683,39 @@ def test_boot_load_respects_local_env_override(monkeypatch): ) assert len(cost_map) > 100 assert get_model_cost_map_source_info()["is_env_forced"] is True + + +def test_boot_load_records_the_file_stamp_and_the_fetch_etag(): + client, _ = _mock_client( + [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_stamped_map_bytes(_STAMP))], + client_cls=httpx.Client, + ) + + cost_map = get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=client) + + assert METADATA_KEY not in cost_map + source = get_model_cost_map_source_info() + assert source["source"] == "remote" + assert source["etag"] == 'W/"boot"' + assert source["generated_at"] == _STAMP["generated_at"] + assert source["source_revision"] == _STAMP["source_revision"] + assert source["loaded_at"] is not None + + +def test_boot_load_fallback_to_the_backup_drops_the_remote_etag(): + """A boot that lands on the bundled backup reports the backup's own stamp and no ETag, even + when an earlier load in the same process had fetched the remote map.""" + remote, _ = _mock_client( + [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_stamped_map_bytes(_STAMP))], + client_cls=httpx.Client, + ) + get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=remote) + failing, _ = _mock_client([httpx.Response(404)], client_cls=httpx.Client) + + cost_map = get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=failing) + + assert METADATA_KEY not in cost_map + source = get_model_cost_map_source_info() + assert source["source"] == "local" + assert source["etag"] is None + assert {"generated_at": source["generated_at"], "source_revision": source["source_revision"]} == _load_bundled_stamp() diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py index b75ee1caccf..fb3583a7dd2 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py @@ -11,6 +11,7 @@ Routes covered: from __future__ import annotations import json +from pathlib import Path from unittest.mock import AsyncMock, MagicMock @@ -20,6 +21,13 @@ from .conftest import VOLATILE_KEYS, normalize # dict-equality assertions remain stable. _VOLATILE = VOLATILE_KEYS | frozenset({"timestamp"}) +_PROVENANCE = { + "generated_at": "2026-09-07T00:00:00Z", + "source_revision": "0123456789abcdef0123456789abcdef01234567", + "etag": 'W/"cost-map-etag"', +} +_ROOT_COST_MAP = Path(__file__).resolve().parents[4] / "model_prices_and_context_window.json" + # --------------------------------------------------------------------------- # Helpers @@ -42,6 +50,14 @@ def _attach_litellm_config(mock_prisma): return table +def _pin_provenance(monkeypatch): + """Fix what this process reports as its cost map revision, independent of the map loaded at import.""" + monkeypatch.setattr( + "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map_provenance", + lambda: dict(_PROVENANCE), + ) + + # --------------------------------------------------------------------------- # POST /reload/model_cost_map # --------------------------------------------------------------------------- @@ -55,6 +71,7 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): table = _attach_litellm_config(mock_prisma) monkeypatch.setattr(ps, "prisma_client", mock_prisma) + _pin_provenance(monkeypatch) fake_cost_map = {"gpt-4": {"input_cost": 0.03}, "gpt-3.5": {"input_cost": 0.002}} monkeypatch.setattr( @@ -83,6 +100,7 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): "status": "success", "models_count": 2, "timestamp": "", + **_PROVENANCE, } assert table.upsert.await_count == 1 update_payload = table.upsert.await_args.kwargs["data"]["update"] @@ -90,6 +108,57 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): assert update_payload["reload_revision"] == {"increment": 1} +def test_reload_model_cost_map_surfaces_provenance_and_keeps_metadata_out_of_the_model_list( + client, auth_as, monkeypatch, mock_prisma +): + """A real refetch through the reload route reports the file's stamp and the fetch ETag on every + status surface, while the ``_metadata`` block never shows up as a model anywhere.""" + import httpx + + import litellm + from litellm.proxy import proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _attach_litellm_config(mock_prisma) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) + stamped = {**json.loads(_ROOT_COST_MAP.read_text()), "_metadata": {k: v for k, v in _PROVENANCE.items() if k != "etag"}} + served = httpx.Response(200, headers={"ETag": _PROVENANCE["etag"]}, content=json.dumps(stamped).encode()) + monkeypatch.setattr( + "litellm.litellm_core_utils.get_model_cost_map._default_reload_client", + lambda: httpx.AsyncClient(transport=httpx.MockTransport(lambda request: served)), + ) + monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None) + monkeypatch.setattr("litellm.model_cost", {}, raising=False) + + async def _fake_invalidate(name): + return None + + monkeypatch.setattr(ps, "invalidate_config_param", _fake_invalidate) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + reload_response = client.post("/reload/model_cost_map") + source_response = client.get("/model/cost_map/source") + status_response = client.get("/schedule/model_cost_map_reload/status") + public_response = client.get("/public/litellm_model_cost_map") + + assert reload_response.status_code == 200 + reload_body = reload_response.json() + assert {key: reload_body[key] for key in _PROVENANCE} == _PROVENANCE + assert source_response.status_code == 200 + source_body = source_response.json() + assert {key: source_body[key] for key in _PROVENANCE} == _PROVENANCE + assert source_body["source"] == "remote" + assert status_response.status_code == 200 + assert {key: status_response.json()[key] for key in _PROVENANCE} == _PROVENANCE + assert public_response.status_code == 200 + public_body = public_response.json() + assert "_metadata" not in public_body + assert "_metadata" not in litellm.model_cost + assert "gpt-4o" in public_body + assert reload_body["models_count"] == len(litellm.model_cost) + + def test_reload_model_cost_map_fetch_failure_502_keeps_map( client, auth_as, monkeypatch, mock_prisma ): @@ -270,11 +339,12 @@ def test_cancel_model_cost_map_reload_no_db_500(client, auth_as, monkeypatch): def test_get_model_cost_map_reload_status_no_db_not_scheduled( client, auth_as, monkeypatch ): - """No prisma client → returns the not-scheduled shape (4 keys, all-null).""" + """No prisma client → returns the not-scheduled shape (all-null) plus the cost map provenance.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles monkeypatch.setattr(ps, "prisma_client", None) + _pin_provenance(monkeypatch) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/schedule/model_cost_map_reload/status") assert response.status_code == 200 @@ -283,6 +353,7 @@ def test_get_model_cost_map_reload_status_no_db_not_scheduled( "interval_hours": None, "last_run": None, "next_run": None, + **_PROVENANCE, } @@ -300,6 +371,7 @@ def test_get_model_cost_map_reload_status_scheduled( config_row.last_run_at = None table.find_unique = AsyncMock(return_value=config_row) monkeypatch.setattr(ps, "prisma_client", mock_prisma) + _pin_provenance(monkeypatch) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/schedule/model_cost_map_reload/status") @@ -309,6 +381,7 @@ def test_get_model_cost_map_reload_status_scheduled( "interval_hours": 12, "last_run": None, "next_run": None, + **_PROVENANCE, } @@ -328,6 +401,7 @@ def test_get_model_cost_map_reload_status_reports_persisted_last_run( config_row.last_run_at = datetime(2024, 1, 1, 6, 0, 0, tzinfo=timezone.utc) table.find_unique = AsyncMock(return_value=config_row) monkeypatch.setattr(ps, "prisma_client", mock_prisma) + _pin_provenance(monkeypatch) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/schedule/model_cost_map_reload/status") @@ -337,6 +411,7 @@ def test_get_model_cost_map_reload_status_reports_persisted_last_run( "interval_hours": 6, "last_run": "2024-01-01T06:00:00+00:00", "next_run": "2024-01-01T12:00:00+00:00", + **_PROVENANCE, } @@ -356,6 +431,7 @@ def test_get_model_cost_map_reload_status_no_config_not_scheduled( config_row.last_run_at = None table.find_unique = AsyncMock(return_value=config_row) monkeypatch.setattr(ps, "prisma_client", mock_prisma) + _pin_provenance(monkeypatch) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/schedule/model_cost_map_reload/status") @@ -365,6 +441,7 @@ def test_get_model_cost_map_reload_status_no_config_not_scheduled( "interval_hours": None, "last_run": None, "next_run": None, + **_PROVENANCE, } @@ -391,6 +468,8 @@ def test_get_model_cost_map_source_happy(client, auth_as, monkeypatch): "url": "https://example.invalid/cost_map.json", "is_env_forced": False, "fallback_reason": None, + "loaded_at": "2026-09-07T01:02:03+00:00", + **_PROVENANCE, } monkeypatch.setattr( "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map_source_info", @@ -406,6 +485,8 @@ def test_get_model_cost_map_source_happy(client, auth_as, monkeypatch): "url": "https://example.invalid/cost_map.json", "is_env_forced": False, "fallback_reason": None, + "loaded_at": "2026-09-07T01:02:03+00:00", + **_PROVENANCE, "model_count": 3, } diff --git a/tests/test_litellm/test_auto_update_price_and_context_window_file.py b/tests/test_litellm/test_auto_update_price_and_context_window_file.py new file mode 100644 index 00000000000..d3cda09cd96 --- /dev/null +++ b/tests/test_litellm/test_auto_update_price_and_context_window_file.py @@ -0,0 +1,54 @@ +"""Tests for .github/scripts/auto_update_price_and_context_window_file.py.""" + +import importlib.util +import json +import re +import sys +from pathlib import Path +from typing import Final + +_REPO_ROOT: Final = Path(__file__).resolve().parents[2] +_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "auto_update_price_and_context_window_file.py" +_spec: Final = importlib.util.spec_from_file_location("auto_update_price_and_context_window_file", _MODULE_PATH) +script: Final = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = script +_spec.loader.exec_module(script) + +_LOCAL_FILE: Final = "model_prices_and_context_window.json" +_GENERATED_AT: Final = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z") + + +def _openrouter_row(model_id: str) -> dict: + return {"id": model_id, "context_length": 8192, "pricing": {"prompt": "0.000001", "completion": "0.000002"}} + + +def _serve(openrouter_rows: list) -> object: + async def fetch_data(url: str) -> list: + return openrouter_rows if "openrouter" in url else [] + + return fetch_data + + +def _read_local(tmp_path: Path) -> dict: + return json.loads((tmp_path / _LOCAL_FILE).read_text()) + + +def test_main_stamps_provenance_only_when_the_sync_changed_the_file(tmp_path: Path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("GITHUB_SHA", "feedface") + monkeypatch.setattr(script, "fetch_data", _serve([_openrouter_row("acme/x")])) + (tmp_path / _LOCAL_FILE).write_text(json.dumps({"sample_spec": {"input_cost_per_token": "USD"}}, indent=4) + "\n") + + script.main() + + written = _read_local(tmp_path) + assert written["openrouter/acme/x"]["litellm_provider"] == "openrouter" + assert written["_metadata"]["source_revision"] == "feedface" + assert _GENERATED_AT.fullmatch(written["_metadata"]["generated_at"]) + + sentinel = {**written, "_metadata": {**written["_metadata"], "generated_at": "2000-01-01T00:00:00Z"}} + (tmp_path / _LOCAL_FILE).write_text(json.dumps(sentinel, indent=4) + "\n") + + script.main() + + assert _read_local(tmp_path) == sentinel diff --git a/tests/test_litellm/test_cost_map_guard.py b/tests/test_litellm/test_cost_map_guard.py index 1b4330ed62c..1a60cf81164 100644 --- a/tests/test_litellm/test_cost_map_guard.py +++ b/tests/test_litellm/test_cost_map_guard.py @@ -141,6 +141,26 @@ def test_bot_may_not_change_special_root_keys() -> None: assert _failures(head) == ("bot PRs may not change fallback_generalizations",) +STAMP: Final = {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef0123456789abcdef01234567"} + + +def test_bot_may_stamp_and_restamp_metadata() -> None: + stamped = _snapshot({**BASE_MAP, "_metadata": STAMP}) + assert _failures(stamped) == () + assert _failures(stamped, bot=False) == () + + restamped = _snapshot( + { + **BASE_MAP, + "_metadata": {**STAMP, "generated_at": "2026-09-14T00:00:00Z"}, + "fallback_generalizations": {"rules": []}, + } + ) + assert guard.guard_failures(stamped, restamped, MAP_FILES, True) == ( + "bot PRs may not change fallback_generalizations", + ) + + def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str: text = _serialize(cost_map) (repo / guard.COST_MAP_PATH).write_text(text) diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index c2c22c25998..3517f5840e8 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -98,6 +98,25 @@ def test_schema_rejects_malformed_entries(committed_schema: dict, entry: dict): assert not validator.is_valid({"some-model": entry}) +@pytest.mark.parametrize( + "metadata", + [ + "2026-09-07T00:00:00Z", + {"generated_at": "2026-09-07T00:00:00Z"}, + {"source_revision": "0123456789abcdef"}, + {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef", "author": "bot"}, + ], + ids=["not_an_object", "missing_revision", "missing_generated_at", "unknown_field"], +) +def test_schema_rejects_a_malformed_metadata_block(committed_schema: dict, metadata: object): + assert not build_validator(committed_schema).is_valid({"_metadata": metadata}) + + +def test_schema_accepts_the_provenance_stamp_as_a_non_model_root_key(committed_schema: dict): + stamp = {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef0123456789abcdef01234567"} + assert build_validator(committed_schema).is_valid({"_metadata": stamp}) + + def test_schema_accepts_minimal_and_unknown_optional_fields(committed_schema: dict): validator = build_validator(committed_schema) assert validator.is_valid({"some-model": {"litellm_provider": "openai"}}) diff --git a/tests/test_litellm/test_sync_together_ai_models.py b/tests/test_litellm/test_sync_together_ai_models.py index b8a85bcfbdc..f58f573c208 100644 --- a/tests/test_litellm/test_sync_together_ai_models.py +++ b/tests/test_litellm/test_sync_together_ai_models.py @@ -1,5 +1,6 @@ import importlib.util import json +import re from pathlib import Path from types import MappingProxyType @@ -369,6 +370,58 @@ def test_sync_is_idempotent_over_the_repo_cost_map() -> None: assert second.cost_map == first.cost_map +def test_stamp_metadata_adds_the_provenance_block_without_touching_models() -> None: + cost_map = {"sample_spec": {"input_cost_per_token": "USD"}, "together_ai/acme/x": {"mode": "chat"}} + + stamped = sync.stamp_metadata(cost_map, "2026-09-07T00:00:00Z", "feedface") + + assert stamped["_metadata"] == {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "feedface"} + assert {key: value for key, value in stamped.items() if key != "_metadata"} == cost_map + assert "_metadata" not in cost_map + + +def _write_registry(repo_root: Path, cost_map: dict) -> None: + for relpath in sync.COST_MAP_RELPATHS: + target = repo_root / relpath + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(cost_map, indent=4) + "\n") + + +def _read_registries(repo_root: Path) -> tuple[dict, ...]: + return tuple(json.loads((repo_root / relpath).read_text()) for relpath in sync.COST_MAP_RELPATHS) + + +def test_write_stamps_provenance_into_both_files_only_when_the_sync_changed_them(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("GITHUB_SHA", "feedface") + cost_map = json.loads((ROOT / "model_prices_and_context_window.json").read_text()) + dropped = next(f"together_ai/{model.id}" for model in RECORDED_CATALOG if f"together_ai/{model.id}" in cost_map) + _write_registry(tmp_path, {key: value for key, value in cost_map.items() if key not in {dropped, "_metadata"}}) + argv = ( + "--write", + "--models-json", + str(FIXTURES / "models_serverless.json"), + "--deprecations-md", + str(FIXTURES / "deprecations.md"), + "--repo-root", + str(tmp_path), + ) + + assert sync.main(argv) == 0 + + written = _read_registries(tmp_path) + assert written[0] == written[1] + assert dropped in written[0] + assert written[0]["_metadata"]["source_revision"] == "feedface" + assert re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", written[0]["_metadata"]["generated_at"]) + + sentinel = {**written[0], "_metadata": {**written[0]["_metadata"], "generated_at": "2000-01-01T00:00:00Z"}} + _write_registry(tmp_path, sentinel) + + assert sync.main(argv) == 0 + + assert _read_registries(tmp_path) == (sentinel, sentinel) + + def test_pr_body_lists_every_section_and_the_skipped_types() -> None: outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) body = sync.render_pr_body(outcome) diff --git a/ui/litellm-dashboard/src/components/price_data_reload.test.tsx b/ui/litellm-dashboard/src/components/price_data_reload.test.tsx index 01381df1620..a85211f498c 100644 --- a/ui/litellm-dashboard/src/components/price_data_reload.test.tsx +++ b/ui/litellm-dashboard/src/components/price_data_reload.test.tsx @@ -32,8 +32,18 @@ const remoteSource = { url: "https://pricing.example.test/model_prices.json", is_env_forced: false, fallback_reason: null, + loaded_at: null, + generated_at: null, + source_revision: null, + etag: null, model_count: 1234, }; +const provenance = { + loaded_at: "2026-09-07T10:00:00Z", + generated_at: "2026-09-06T23:38:47Z", + source_revision: "cd681a573fd9f5b6f15a1355f46178e4e9d374d2", + etag: 'W/"eb8e9a53f4cc284b"', +}; describe("PriceDataReload", () => { beforeEach(() => { @@ -51,6 +61,30 @@ describe("PriceDataReload", () => { expect(screen.getByText("No periodic reload scheduled")).toBeInTheDocument(); }); + it("shows which revision of the cost map is loaded when the source reports one", async () => { + vi.mocked(getModelCostMapSource).mockResolvedValue({ ...remoteSource, ...provenance } as never); + render(); + + expect(await screen.findByText("Source revision:")).toBeInTheDocument(); + expect(screen.getByText("cd681a573fd9")).toBeInTheDocument(); + expect(screen.getByText("ETag:")).toBeInTheDocument(); + expect(screen.getByText('W/"eb8e9a53f4cc284b"')).toBeInTheDocument(); + expect(screen.getByText("Generated at:")).toBeInTheDocument(); + expect(screen.getByText(new Date(provenance.generated_at).toLocaleString())).toBeInTheDocument(); + expect(screen.getByText("Loaded at:")).toBeInTheDocument(); + expect(screen.getByText(new Date(provenance.loaded_at).toLocaleString())).toBeInTheDocument(); + }); + + it("hides the provenance rows when the loaded map carries no stamp", async () => { + render(); + + expect(await screen.findByText("Pricing Data Source")).toBeInTheDocument(); + expect(screen.queryByText("Generated at:")).not.toBeInTheDocument(); + expect(screen.queryByText("Source revision:")).not.toBeInTheDocument(); + expect(screen.queryByText("ETag:")).not.toBeInTheDocument(); + expect(screen.queryByText("Loaded at:")).not.toBeInTheDocument(); + }); + it("confirms an immediate reload and refreshes dependent data", async () => { const user = userEvent.setup(); const onReloadSuccess = vi.fn(); diff --git a/ui/litellm-dashboard/src/components/price_data_reload.tsx b/ui/litellm-dashboard/src/components/price_data_reload.tsx index 1c6801eede2..3bb70072937 100644 --- a/ui/litellm-dashboard/src/components/price_data_reload.tsx +++ b/ui/litellm-dashboard/src/components/price_data_reload.tsx @@ -49,9 +49,17 @@ interface CostMapSourceInfo { url: string | null; is_env_forced: boolean; fallback_reason: string | null; + loaded_at: string | null; + generated_at: string | null; + source_revision: string | null; + etag: string | null; model_count: number; } +const SHORT_REVISION_LENGTH = 12; + +const shortRevision = (revision: string) => revision.slice(0, SHORT_REVISION_LENGTH); + const EMPTY_RELOAD_STATUS: ReloadStatus = { scheduled: false, interval_hours: null, @@ -89,6 +97,55 @@ const isValidReloadInterval = (value: number) => { return value >= 1 && value <= 168; }; +const formatDateTime = (dateTimeString: string | null) => { + if (!dateTimeString) return "Never"; + try { + return new Date(dateTimeString).toLocaleString(); + } catch { + return dateTimeString; + } +}; + +const CostMapProvenanceRows: React.FC<{ sourceInfo: CostMapSourceInfo }> = ({ sourceInfo }) => ( + <> + {sourceInfo.generated_at && ( +
+ Generated at: + {formatDateTime(sourceInfo.generated_at)} +
+ )} + + {sourceInfo.source_revision && ( +
+ Source revision: + + }> + {shortRevision(sourceInfo.source_revision)} + + {sourceInfo.source_revision} + +
+ )} + + {sourceInfo.etag && ( +
+ ETag: + + }>{sourceInfo.etag} + {sourceInfo.etag} + +
+ )} + + {sourceInfo.loaded_at && ( +
+ Loaded at: + {formatDateTime(sourceInfo.loaded_at)} +
+ )} + +); + const PriceDataReload: React.FC = ({ accessToken, onReloadSuccess, @@ -227,15 +284,6 @@ const PriceDataReload: React.FC = ({ } }; - const formatDateTime = (dateTimeString: string | null) => { - if (!dateTimeString) return "Never"; - try { - return new Date(dateTimeString).toLocaleString(); - } catch { - return dateTimeString; - } - }; - const getStatusText = () => { if (!reloadStatus?.scheduled) return "Not scheduled"; if (!reloadStatus.last_run) return "Ready"; @@ -334,6 +382,8 @@ const PriceDataReload: React.FC = ({ )} + + {sourceInfo.is_env_forced && (
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6fb08445aff..7b9b24c9627 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8684,6 +8684,9 @@ export interface paths { * - url: the remote URL that was attempted (null when env-forced local) * - is_env_forced: true if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage * - fallback_reason: human-readable reason why remote failed (null on success) + * - loaded_at: when this pod last loaded the map + * - generated_at, source_revision: the _metadata stamp inside the loaded file + * - etag: the ETag of the remote fetch (null for the bundled backup) * - model_count: number of models in the currently loaded cost map */ get: operations["get_model_cost_map_source_model_cost_map_source_get"]; From 6c1bba54c2c2d8e2b8c47673678eccdcda4f36a2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:28:58 -0700 Subject: [PATCH 02/14] fix(ui): show a malformed generated_at stamp as-is on the Price Data Reload card --- .../src/components/price_data_reload.test.tsx | 13 +++++++++++++ .../src/components/price_data_reload.tsx | 7 ++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/components/price_data_reload.test.tsx b/ui/litellm-dashboard/src/components/price_data_reload.test.tsx index a85211f498c..fde69675b72 100644 --- a/ui/litellm-dashboard/src/components/price_data_reload.test.tsx +++ b/ui/litellm-dashboard/src/components/price_data_reload.test.tsx @@ -75,6 +75,19 @@ describe("PriceDataReload", () => { expect(screen.getByText(new Date(provenance.loaded_at).toLocaleString())).toBeInTheDocument(); }); + it("shows a malformed generated_at stamp as-is instead of Invalid Date", async () => { + vi.mocked(getModelCostMapSource).mockResolvedValue({ + ...remoteSource, + ...provenance, + generated_at: "yesterday-ish", + } as never); + render(); + + expect(await screen.findByText("Generated at:")).toBeInTheDocument(); + expect(screen.getByText("yesterday-ish")).toBeInTheDocument(); + expect(screen.queryByText("Invalid Date")).not.toBeInTheDocument(); + }); + it("hides the provenance rows when the loaded map carries no stamp", async () => { render(); diff --git a/ui/litellm-dashboard/src/components/price_data_reload.tsx b/ui/litellm-dashboard/src/components/price_data_reload.tsx index 3bb70072937..c152916f271 100644 --- a/ui/litellm-dashboard/src/components/price_data_reload.tsx +++ b/ui/litellm-dashboard/src/components/price_data_reload.tsx @@ -99,11 +99,8 @@ const isValidReloadInterval = (value: number) => { const formatDateTime = (dateTimeString: string | null) => { if (!dateTimeString) return "Never"; - try { - return new Date(dateTimeString).toLocaleString(); - } catch { - return dateTimeString; - } + const parsed = new Date(dateTimeString); + return Number.isNaN(parsed.getTime()) ? dateTimeString : parsed.toLocaleString(); }; const CostMapProvenanceRows: React.FC<{ sourceInfo: CostMapSourceInfo }> = ({ sourceInfo }) => ( From aa1c76bc3b601e8beef987101297a9e7f94f8560 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:28:59 -0700 Subject: [PATCH 03/14] test(cost_map): skip every reserved top-level key in the price map schema test --- tests/test_litellm/test_utils.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 8a56a84ade7..f6c9a4537a1 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -21,6 +21,7 @@ from litellm._logging import ( verbose_logger, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.get_model_cost_map import RESERVED_TOP_LEVEL_KEYS from litellm.proxy.utils import is_valid_api_key from litellm.types.utils import ( CallTypes, @@ -1218,15 +1219,12 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): with open(prod_json, "r") as model_prices_file: actual_json = json.load(model_prices_file) assert isinstance(actual_json, dict) - actual_json.pop( - "sample_spec", None - ) # remove the sample, whose schema is inconsistent with the real data - actual_json.pop( - "fallback_generalizations", None - ) # reserved meta key, not a model entry + model_entries: Final = { + key: value for key, value in actual_json.items() if key not in RESERVED_TOP_LEVEL_KEYS + } # Validate schema - validate(actual_json, INTENDED_SCHEMA) + validate(model_entries, INTENDED_SCHEMA) # Validate cost values # Define exceptions for models that are allowed to have costs > 1 @@ -1237,7 +1235,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "runwayml/seedance2", # 4K output is 150 credits/second = $1.50/second ] - is_valid, violations = validate_model_cost_values(actual_json, exceptions) + is_valid, violations = validate_model_cost_values(model_entries, exceptions) if not is_valid: error_message = "Cost validation failed:\n" + "\n".join(violations) @@ -1268,8 +1266,7 @@ def test_max_tokens_consistency(): inconsistencies = [] for model_name, config in models.items(): - # Skip the sample_spec - if model_name == "sample_spec": + if model_name in RESERVED_TOP_LEVEL_KEYS: continue # Check if both max_tokens and max_output_tokens exist From 9041768fb43715dc8c28e5dc139c86adac2659ce Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:47:51 -0700 Subject: [PATCH 04/14] feat(cost_map): derive source_revision from the loaded bytes instead of a _metadata stamp The revision an operator checks is now the git blob id of the exact bytes the process loaded, the same id git rev-parse :model_prices_and_context_window.json prints, so it is always present, never goes stale between bot writes, and needs no stamp in the JSON that every PR touching the file would have to regenerate. The _metadata block, the generated_at field, the schema and guard changes, and the bot stamping are dropped --- ...to_update_price_and_context_window_file.py | 27 +--- ci_cd/cost_map_guard.py | 7 +- ci_cd/generate_model_prices_schema.py | 19 +-- .../litellm_core_utils/get_model_cost_map.py | 98 ++++++------- ...odel_prices_and_context_window_backup.json | 4 - litellm/proxy/proxy_server.py | 2 +- model_prices_and_context_window.json | 4 - model_prices_and_context_window.schema.json | 20 +-- scripts/sync_together_ai_models.py | 23 +-- .../test_get_model_cost_map.py | 135 +++++++----------- .../test_routes_model_cost_map.py | 24 ++-- ...to_update_price_and_context_window_file.py | 54 ------- tests/test_litellm/test_cost_map_guard.py | 20 --- .../test_litellm/test_model_prices_schema.py | 19 --- .../test_sync_together_ai_models.py | 53 ------- tests/test_litellm/test_utils.py | 17 ++- .../src/components/price_data_reload.test.tsx | 15 +- .../src/components/price_data_reload.tsx | 8 -- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 19 files changed, 131 insertions(+), 420 deletions(-) delete mode 100644 tests/test_litellm/test_auto_update_price_and_context_window_file.py diff --git a/.github/scripts/auto_update_price_and_context_window_file.py b/.github/scripts/auto_update_price_and_context_window_file.py index a7a3194f262..461d8d347d9 100644 --- a/.github/scripts/auto_update_price_and_context_window_file.py +++ b/.github/scripts/auto_update_price_and_context_window_file.py @@ -1,9 +1,6 @@ import asyncio import aiohttp import json -import os -import subprocess -from datetime import datetime, timezone # Asynchronously fetch data from a given URL async def fetch_data(url): @@ -34,28 +31,13 @@ def sync_local_data_with_remote(local_data, remote_data): for key in (set(remote_data) - set(local_data)): local_data[key] = remote_data[key] -def utc_now_iso(): - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def source_revision(): - from_env = os.environ.get("GITHUB_SHA") - if from_env: - return from_env - return subprocess.run(["git", "rev-parse", "HEAD"], check=True, capture_output=True, text=True).stdout.strip() - - -def stamp_metadata(data, generated_at, revision): - return {**data, "_metadata": {"generated_at": generated_at, "source_revision": revision}} - - # Write data to the json file def write_to_file(file_path, data): try: # Open the file in write mode with open(file_path, "w") as file: # Dump the data as JSON into the file - file.write(json.dumps(data, indent=4) + "\n") + json.dump(data, file, indent=4) print("Values updated successfully.") except Exception as e: # Print an error message if writing to file fails @@ -167,13 +149,8 @@ def main(): # If both local and openrouter data are available, synchronize and save if local_data and all_remote_data: - before = json.dumps(local_data, sort_keys=True) sync_local_data_with_remote(local_data, all_remote_data) - changed = json.dumps(local_data, sort_keys=True) != before - write_to_file( - local_file_path, - stamp_metadata(local_data, utc_now_iso(), source_revision()) if changed else local_data, - ) + write_to_file(local_file_path, local_data) else: print("Failed to fetch model data from either local file or URL.") diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 351c06c74eb..50aa40ba220 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -2,8 +2,7 @@ Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named -litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models, plus -restamp the _metadata provenance block. +litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models. """ from __future__ import annotations @@ -16,7 +15,7 @@ from collections.abc import Sequence from dataclasses import dataclass from typing import Final -from generate_model_prices_schema import BOT_LOCKED_ROOT_KEYS, build_schema, render, validation_errors +from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors COST_MAP_PATH: Final = "model_prices_and_context_window.json" BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" @@ -103,7 +102,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str *(f"bot PRs may not remove fields: {ref}" for ref in removed_fields), *( f"bot PRs may not change {key}" - for key in sorted(BOT_LOCKED_ROOT_KEYS) + for key in sorted(SPECIAL_ROOT_KEYS) if base_map.get(key) != head_map.get(key) ), ) diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 557afa50128..ab29b70bdd4 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -11,9 +11,7 @@ REPO_ROOT = Path(__file__).parent.parent PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json" -METADATA_KEY = "_metadata" -SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations", METADATA_KEY}) -BOT_LOCKED_ROOT_KEYS = SPECIAL_ROOT_KEYS - {METADATA_KEY} +SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"}) JsonSchema = dict @@ -273,26 +271,13 @@ def build_schema(prices: dict) -> JsonSchema: "description": ( "Schema for LiteLLM's model price and context window registry " "(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). " - "Every top-level key except '_metadata', 'sample_spec', and 'fallback_generalizations' is a model id, " + "Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, " "optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. " "All costs are USD per unit. New optional fields are added regularly, so consumers should " "ignore unknown fields rather than reject them." ), "type": "object", "properties": { - METADATA_KEY: { - "type": "object", - "description": ( - "Provenance of this file: when an automated sync last regenerated it and the commit it " - "ran against. Human edits leave it untouched; not a model entry." - ), - "properties": { - "generated_at": {"type": "string", "format": "date-time"}, - "source_revision": STRING, - }, - "required": ["generated_at", "source_revision"], - "additionalProperties": False, - }, "sample_spec": { "type": "object", "description": ( diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index a538e7cb330..2bdfbc66088 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -9,18 +9,18 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True """ import asyncio +import hashlib import json import os import random import time from collections.abc import Awaitable, Callable -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime, timezone from importlib.resources import files from typing import Final, Protocol import httpx -from pydantic import BaseModel, ConfigDict, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm import verbose_logger @@ -33,11 +33,10 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" -METADATA_KEY: Final = "_metadata" # Reserved top-level keys that are not model entries. They must be excluded # from the model-count integrity check so a real upstream shrink can't be masked. -RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY, METADATA_KEY}) +RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY}) def _count_model_entries(model_cost: dict) -> int: @@ -45,6 +44,11 @@ def _count_model_entries(model_cost: dict) -> int: return sum(1 for key in model_cost if key not in RESERVED_TOP_LEVEL_KEYS) +def git_blob_id(body: bytes) -> str: + """The sha1 git gives these bytes as a blob, so ``git rev-parse :`` reproduces it for the file""" + return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest() + + class GetModelCostMap: """ Handles fetching, validating, and loading the model cost map. @@ -56,15 +60,25 @@ class GetModelCostMap: _backup_model_count: int = -1 # -1 = not yet loaded + @staticmethod + def read_local_model_cost_map_bytes() -> bytes: + return files("litellm").joinpath("model_prices_and_context_window_backup.json").read_bytes() + @staticmethod def read_local_model_cost_map_text() -> str: - return files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text(encoding="utf-8") + return GetModelCostMap.read_local_model_cost_map_bytes().decode("utf-8") + + @staticmethod + def load_local_model_cost_map_with_revision() -> "ModelCostMapReloaded": + """The bundled backup map together with the git blob id of the file it was parsed from""" + body: Final = GetModelCostMap.read_local_model_cost_map_bytes() + content: Final = json.loads(body) + return ModelCostMapReloaded(model_cost_map=content, revision=git_blob_id(body)) @staticmethod def load_local_model_cost_map() -> dict: """Load the local backup model cost map bundled with the package.""" - content: Final = json.loads(GetModelCostMap.read_local_model_cost_map_text()) - return content + return GetModelCostMap.load_local_model_cost_map_with_revision().model_cost_map @classmethod def _get_backup_model_count(cls) -> int: @@ -169,6 +183,7 @@ MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0 @dataclass(frozen=True, slots=True) class ModelCostMapReloaded: model_cost_map: dict # mutable-ok: adopted as litellm.model_cost, whose consumer contract is a plain mutable dict + revision: str | None = None etag: str | None = None @@ -258,7 +273,9 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}") if not isinstance(parsed, dict): return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}") - return ModelCostMapReloaded(model_cost_map=parsed, etag=response.headers.get("etag")) + return ModelCostMapReloaded( + model_cost_map=parsed, revision=git_blob_id(response.content), etag=response.headers.get("etag") + ) def _next_retry_wait( @@ -337,10 +354,7 @@ async def refetch_model_cost_map( _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True _cost_map_source_info.fallback_reason = None - _cost_map_source_info.etag = None - return ModelCostMapReloaded( - model_cost_map=_finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) - ) + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()) result: Final = await _fetch_remote_model_cost_map_with_retry( url=url, @@ -366,8 +380,7 @@ async def refetch_model_cost_map( _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False _cost_map_source_info.fallback_reason = None - _cost_map_source_info.etag = result.etag - return ModelCostMapReloaded(model_cost_map=_finalize_model_cost_map(result.model_cost_map), etag=result.etag) + return _finalize_loaded_model_cost_map(result) class ModelCostMapSourceInfo: @@ -378,7 +391,6 @@ class ModelCostMapSourceInfo: is_env_forced: bool = False fallback_reason: str | None = None loaded_at: "datetime | None" = None - generated_at: str | None = None source_revision: str | None = None etag: str | None = None @@ -387,28 +399,7 @@ class ModelCostMapSourceInfo: _cost_map_source_info: Final = ModelCostMapSourceInfo() -class CostMapMetadata(BaseModel): - model_config = ConfigDict(frozen=True, extra="ignore") - - generated_at: str | None = None - source_revision: str | None = None - - -_EMPTY_METADATA: Final = CostMapMetadata() - - -def _parse_metadata(raw: object) -> CostMapMetadata: - if raw is None: - return _EMPTY_METADATA - try: - return CostMapMetadata.model_validate(raw) - except ValidationError as error: - verbose_logger.warning("LiteLLM: ignoring a malformed %s block in the model cost map: %s", METADATA_KEY, error) - return _EMPTY_METADATA - - class CostMapProvenance(TypedDict): - generated_at: ReadOnly[str | None] source_revision: ReadOnly[str | None] etag: ReadOnly[str | None] @@ -422,10 +413,10 @@ class CostMapSourceInfo(CostMapProvenance): def get_model_cost_map_provenance() -> CostMapProvenance: - """Which revision of the cost map this process serves: the ``_metadata`` stamp the file - carries plus the ETag the remote fetch returned (None for the bundled backup)""" + """Which revision of the cost map this process serves: the git blob id of the bytes it loaded, the + same id ``git rev-parse :model_prices_and_context_window.json`` prints for a checkout, plus + the ETag the remote fetch returned (None for the bundled backup)""" return { - "generated_at": _cost_map_source_info.generated_at, "source_revision": _cost_map_source_info.source_revision, "etag": _cost_map_source_info.etag, } @@ -441,7 +432,7 @@ def get_model_cost_map_source_info() -> CostMapSourceInfo: - is_env_forced: True if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage - fallback_reason: human-readable reason if remote failed and local was used - loaded_at: ISO 8601 time this process last loaded the map - - generated_at, source_revision: the ``_metadata`` stamp inside the loaded file + - source_revision: git blob id of the loaded file's bytes - etag: the ETag of the remote fetch (None for the bundled backup) """ loaded_at: Final = _cost_map_source_info.loaded_at @@ -451,7 +442,6 @@ def get_model_cost_map_source_info() -> CostMapSourceInfo: "is_env_forced": _cost_map_source_info.is_env_forced, "fallback_reason": _cost_map_source_info.fallback_reason, "loaded_at": loaded_at.isoformat() if loaded_at is not None else None, - "generated_at": _cost_map_source_info.generated_at, "source_revision": _cost_map_source_info.source_revision, "etag": _cost_map_source_info.etag, } @@ -518,21 +508,24 @@ def _expand_model_aliases(model_cost: dict) -> dict: def _finalize_model_cost_map(model_cost: dict) -> dict: - """Extract fallback generalizations and the provenance stamp out of the raw map, then expand aliases. + """Extract fallback generalizations out of the raw map, then expand aliases. The ``fallback_generalizations`` block is installed into the generalizations - module and the ``_metadata`` block into the source info; both are removed from - the map so neither is ever treated as a model entry. + module and removed from the map so it is never treated as a model entry. """ raw: Final = model_cost.pop(FALLBACK_GENERALIZATIONS_KEY, None) rules: Final = raw.get("rules") if isinstance(raw, dict) else None set_fallback_generalizations(rules) - metadata: Final = _parse_metadata(model_cost.pop(METADATA_KEY, None)) - _cost_map_source_info.generated_at = metadata.generated_at - _cost_map_source_info.source_revision = metadata.source_revision return _expand_model_aliases(model_cost) +def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded: + """Record which bytes this process now serves, then finalize the map they parsed into""" + _cost_map_source_info.source_revision = loaded.revision + _cost_map_source_info.etag = loaded.etag + return replace(loaded, model_cost_map=_finalize_model_cost_map(loaded.model_cost_map)) + + def get_model_cost_map( url: str, timeout: int = 5, @@ -561,12 +554,10 @@ def get_model_cost_map( _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True _cost_map_source_info.fallback_reason = None - _cost_map_source_info.etag = None - return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False - _cost_map_source_info.etag = None result: Final = _fetch_remote_model_cost_map_with_retry_sync( url=url, @@ -584,7 +575,7 @@ def get_model_cost_map( ) _cost_map_source_info.source = "local" _cost_map_source_info.fallback_reason = f"Remote fetch failed: {result.reason}" - return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map content: Final = result.model_cost_map # Validate using cached count (cheap int comparison, no file I/O) @@ -598,9 +589,8 @@ def get_model_cost_map( ) _cost_map_source_info.source = "local" _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" - return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map _cost_map_source_info.source = "remote" _cost_map_source_info.fallback_reason = None - _cost_map_source_info.etag = result.etag - return _finalize_model_cost_map(content) + return _finalize_loaded_model_cost_map(result).model_cost_map diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5edb3c0e9d8..b1ffc1583e4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1,8 +1,4 @@ { - "_metadata": { - "generated_at": "2026-09-07T23:38:47Z", - "source_revision": "cd681a573fd9f5b6f15a1355f46178e4e9d374d2" - }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4741e4cd9d3..818a1506754 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -17938,7 +17938,7 @@ async def get_model_cost_map_source( - is_env_forced: true if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage - fallback_reason: human-readable reason why remote failed (null on success) - loaded_at: when this pod last loaded the map - - generated_at, source_revision: the _metadata stamp inside the loaded file + - source_revision: git blob id of the loaded file, what git rev-parse : prints for it - etag: the ETag of the remote fetch (null for the bundled backup) - model_count: number of models in the currently loaded cost map """ diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5edb3c0e9d8..b1ffc1583e4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,8 +1,4 @@ { - "_metadata": { - "generated_at": "2026-09-07T23:38:47Z", - "source_revision": "cd681a573fd9f5b6f15a1355f46178e4e9d374d2" - }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index c40c2a67682..47a1934a703 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -1,27 +1,9 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "title": "LiteLLM model_prices_and_context_window.json", - "description": "Schema for LiteLLM's model price and context window registry (https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Every top-level key except '_metadata', 'sample_spec', and 'fallback_generalizations' is a model id, optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. All costs are USD per unit. New optional fields are added regularly, so consumers should ignore unknown fields rather than reject them.", + "description": "Schema for LiteLLM's model price and context window registry (https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. All costs are USD per unit. New optional fields are added regularly, so consumers should ignore unknown fields rather than reject them.", "type": "object", "properties": { - "_metadata": { - "type": "object", - "description": "Provenance of this file: when an automated sync last regenerated it and the commit it ran against. Human edits leave it untouched; not a model entry.", - "properties": { - "generated_at": { - "type": "string", - "format": "date-time" - }, - "source_revision": { - "type": "string" - } - }, - "required": [ - "generated_at", - "source_revision" - ], - "additionalProperties": false - }, "sample_spec": { "type": "object", "description": "Documentation placeholder illustrating the entry shape; not a real model and not schema-conformant (several values are prose)." diff --git a/scripts/sync_together_ai_models.py b/scripts/sync_together_ai_models.py index e009f1a7ce6..12b128890f1 100644 --- a/scripts/sync_together_ai_models.py +++ b/scripts/sync_together_ai_models.py @@ -19,11 +19,9 @@ import argparse import json import os import re -import subprocess import sys from collections.abc import Mapping, Sequence from dataclasses import dataclass, field -from datetime import datetime, timezone from pathlib import Path from types import MappingProxyType from typing import Final @@ -35,7 +33,6 @@ MODELS_URL: Final = "https://api.together.ai/v1/models?serverless" DEPRECATIONS_URL: Final = "https://docs.together.ai/docs/deprecations.md" PROVIDER: Final = "together_ai" PREFIX: Final = "together_ai/" -METADATA_KEY: Final = "_metadata" SOURCE_URL: Final = "https://docs.together.ai/docs/serverless-models" COST_MAP_RELPATHS: Final = ( "model_prices_and_context_window.json", @@ -498,23 +495,6 @@ def _serialize(cost_map: CostMap) -> str: return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n" -def stamp_metadata(cost_map: CostMap, generated_at: str, source_revision: str) -> CostMap: - return {**cost_map, METADATA_KEY: {"generated_at": generated_at, "source_revision": source_revision}} - - -def _utc_now_iso() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def _source_revision(repo_root: Path) -> str: - from_env: Final = os.environ.get("GITHUB_SHA") - if from_env: - return from_env - return subprocess.run( - ("git", "rev-parse", "HEAD"), cwd=repo_root, check=True, capture_output=True, text=True - ).stdout.strip() - - def main(argv: Sequence[str]) -> int: parser: Final = argparse.ArgumentParser(description=__doc__) parser.add_argument("--write", action="store_true", help="apply the sync to the cost map files (default: dry run)") @@ -547,9 +527,8 @@ def main(argv: Sequence[str]) -> int: if args.pr_body_file is not None: args.pr_body_file.write_text(body) if args.write and outcome.has_changes: - stamped: Final = _serialize(stamp_metadata(outcome.cost_map, _utc_now_iso(), _source_revision(args.repo_root))) for relpath in COST_MAP_RELPATHS: - (args.repo_root / relpath).write_text(stamped) + (args.repo_root / relpath).write_text(_serialize(outcome.cost_map)) print(render_summary(outcome)) print() print(body) diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 62f72495491..d9fe6d2f979 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -17,11 +17,11 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) from litellm.litellm_core_utils.get_model_cost_map import ( FALLBACK_GENERALIZATIONS_KEY, - METADATA_KEY, GetModelCostMap, _count_model_entries, _finalize_model_cost_map, get_model_cost_map_provenance, + git_blob_id, ) @@ -33,18 +33,16 @@ def _load_root_cost_map() -> dict: return json.load(f) -def _load_bundled_stamp() -> dict: - path = os.path.join( - os.path.dirname(__file__), "../../../litellm/model_prices_and_context_window_backup.json" - ) - with open(path) as f: - return json.load(f)[METADATA_KEY] +def _bundled_blob_id() -> str: + path = os.path.join(os.path.dirname(__file__), "../../../litellm/model_prices_and_context_window_backup.json") + with open(path, "rb") as f: + return git_blob_id(f.read()) -_STAMP = { - "generated_at": "2026-09-07T00:00:00Z", - "source_revision": "0123456789abcdef0123456789abcdef01234567", -} +def test_git_blob_id_is_what_git_hash_object_prints(): + """An operator checks a reported revision with ``git hash-object`` or ``git rev-parse :``, + so the id must be git's blob sha1 of the exact bytes, not a plain sha1 or a hash of the parsed JSON.""" + assert git_blob_id(b'{"gpt-5.4-mini": {"mode": "chat"}}\n') == "18b9a8381e13a3b38a2128f184f631f95829e987" def _make_models(n: int) -> dict: @@ -57,7 +55,6 @@ def test_count_model_entries_excludes_reserved_keys(): m = _make_models(3) m["sample_spec"] = {"foo": "bar"} m[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []} - m[METADATA_KEY] = dict(_STAMP) assert _count_model_entries(m) == 3 @@ -143,39 +140,6 @@ def test_finalize_with_no_block_clears_rules(): set_fallback_generalizations(previous) -def test_finalize_pops_metadata_and_records_provenance(): - finalized = _finalize_model_cost_map({**_make_models(2), METADATA_KEY: dict(_STAMP)}) - - assert METADATA_KEY not in finalized - assert len(finalized) == 2 - provenance = get_model_cost_map_provenance() - assert provenance["generated_at"] == _STAMP["generated_at"] - assert provenance["source_revision"] == _STAMP["source_revision"] - - -def test_finalize_without_metadata_clears_the_previous_stamp(): - _finalize_model_cost_map({**_make_models(2), METADATA_KEY: dict(_STAMP)}) - - _finalize_model_cost_map(_make_models(2)) - - provenance = get_model_cost_map_provenance() - assert provenance["generated_at"] is None - assert provenance["source_revision"] is None - - -@pytest.mark.parametrize( - "raw", - ["2026-09-07T00:00:00Z", {"generated_at": 42}, ["2026-09-07T00:00:00Z"]], - ids=["string", "wrong_field_type", "list"], -) -def test_finalize_tolerates_a_malformed_metadata_block(raw): - finalized = _finalize_model_cost_map({**_make_models(2), METADATA_KEY: raw}) - - assert METADATA_KEY not in finalized - assert len(finalized) == 2 - assert get_model_cost_map_provenance()["generated_at"] is None - - def test_shipped_backup_carries_the_claude_routing_rules(): """The bundled backup must ship the Claude routing rules so a fresh install (or an offline fallback) routes unknown Claude models without code changes. @@ -390,10 +354,6 @@ def _real_map_bytes() -> bytes: return json.dumps(_load_root_cost_map()).encode() -def _stamped_map_bytes(stamp: dict) -> bytes: - return json.dumps({**_load_root_cost_map(), METADATA_KEY: stamp}).encode() - - class _SleepRecorder: """Injected in place of asyncio.sleep so tests assert waits without real delay.""" @@ -555,40 +515,51 @@ async def test_refetch_respects_local_env_override(monkeypatch): @pytest.mark.asyncio -async def test_refetch_records_the_file_stamp_and_the_fetch_etag(): - """A reload reports which revision of the map it swapped in: the ``_metadata`` stamp the file - carries plus the ETag the fetch returned, with the stamp itself kept out of the model map.""" - client, _ = _mock_client( - [httpx.Response(200, headers={"ETag": 'W/"abc123"'}, content=_stamped_map_bytes(_STAMP))] - ) +async def test_refetch_records_the_blob_id_of_the_bytes_served_and_the_fetch_etag(): + """A reload reports which revision of the map it swapped in: the git blob id of the exact bytes the + fetch returned, so ``git rev-parse :model_prices_and_context_window.json`` can confirm it, + plus the ETag the fetch returned.""" + body = _real_map_bytes() + client, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"abc123"'}, content=body)]) result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloaded) + assert result.revision == git_blob_id(body) assert result.etag == 'W/"abc123"' - assert METADATA_KEY not in result.model_cost_map - assert get_model_cost_map_provenance() == { - "generated_at": _STAMP["generated_at"], - "source_revision": _STAMP["source_revision"], - "etag": 'W/"abc123"', - } + assert get_model_cost_map_provenance() == {"source_revision": git_blob_id(body), "etag": 'W/"abc123"'} @pytest.mark.asyncio -async def test_refetch_local_override_reports_the_bundled_stamp_without_an_etag(monkeypatch): - """Forcing the bundled backup after a remote reload must drop the remote ETag, since the map - served is no longer the one that ETag identifies.""" - remote, _ = _mock_client( - [httpx.Response(200, headers={"ETag": 'W/"remote"'}, content=_stamped_map_bytes(_STAMP))] +async def test_refetch_revision_follows_the_bytes_not_the_url(): + """Two fetches of the same URL that return different bytes report different revisions.""" + edited = json.loads(_real_map_bytes()) + edited["gpt-5.4-mini"]["input_cost_per_token"] = 0.5 + client, _ = _mock_client( + [httpx.Response(200, content=_real_map_bytes()), httpx.Response(200, content=json.dumps(edited).encode())] ) + + first = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) + second = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) + + assert isinstance(first, ModelCostMapReloaded) and isinstance(second, ModelCostMapReloaded) + assert first.revision != second.revision + assert get_model_cost_map_provenance()["source_revision"] == second.revision + + +@pytest.mark.asyncio +async def test_refetch_local_override_reports_the_bundled_blob_id_without_an_etag(monkeypatch): + """Forcing the bundled backup after a remote reload must report the backup's own blob id and drop the + remote ETag, since the map served is no longer the one that ETag identifies.""" + remote, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"remote"'}, content=_real_map_bytes())]) await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=remote) monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0)) assert isinstance(result, ModelCostMapReloaded) - assert METADATA_KEY not in result.model_cost_map - assert get_model_cost_map_provenance() == {**_load_bundled_stamp(), "etag": None} + assert result.revision == _bundled_blob_id() + assert get_model_cost_map_provenance() == {"source_revision": _bundled_blob_id(), "etag": None} # --------------------------------------------------------------------------- @@ -633,7 +604,7 @@ def test_boot_load_retries_transient_failures_instead_of_falling_back(): source = get_model_cost_map_source_info() assert source["source"] == "remote" assert source["fallback_reason"] is None - assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY, METADATA_KEY} + assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} def test_boot_load_honors_retry_after_then_falls_back_after_max_attempts(): @@ -685,37 +656,31 @@ def test_boot_load_respects_local_env_override(monkeypatch): assert get_model_cost_map_source_info()["is_env_forced"] is True -def test_boot_load_records_the_file_stamp_and_the_fetch_etag(): - client, _ = _mock_client( - [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_stamped_map_bytes(_STAMP))], - client_cls=httpx.Client, - ) +def test_boot_load_records_the_blob_id_of_the_bytes_served_and_the_fetch_etag(): + body = _real_map_bytes() + client, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=body)], client_cls=httpx.Client) - cost_map = get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=client) + get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=client) - assert METADATA_KEY not in cost_map source = get_model_cost_map_source_info() assert source["source"] == "remote" assert source["etag"] == 'W/"boot"' - assert source["generated_at"] == _STAMP["generated_at"] - assert source["source_revision"] == _STAMP["source_revision"] + assert source["source_revision"] == git_blob_id(body) assert source["loaded_at"] is not None -def test_boot_load_fallback_to_the_backup_drops_the_remote_etag(): - """A boot that lands on the bundled backup reports the backup's own stamp and no ETag, even +def test_boot_load_fallback_to_the_backup_reports_its_blob_id_and_drops_the_remote_etag(): + """A boot that lands on the bundled backup reports the backup's own blob id and no ETag, even when an earlier load in the same process had fetched the remote map.""" remote, _ = _mock_client( - [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_stamped_map_bytes(_STAMP))], - client_cls=httpx.Client, + [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_real_map_bytes())], client_cls=httpx.Client ) get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=remote) failing, _ = _mock_client([httpx.Response(404)], client_cls=httpx.Client) - cost_map = get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=failing) + get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=failing) - assert METADATA_KEY not in cost_map source = get_model_cost_map_source_info() assert source["source"] == "local" assert source["etag"] is None - assert {"generated_at": source["generated_at"], "source_revision": source["source_revision"]} == _load_bundled_stamp() + assert source["source_revision"] == _bundled_blob_id() diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py index fb3583a7dd2..0490993a314 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py @@ -22,7 +22,6 @@ from .conftest import VOLATILE_KEYS, normalize _VOLATILE = VOLATILE_KEYS | frozenset({"timestamp"}) _PROVENANCE = { - "generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef0123456789abcdef01234567", "etag": 'W/"cost-map-etag"', } @@ -108,22 +107,24 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): assert update_payload["reload_revision"] == {"increment": 1} -def test_reload_model_cost_map_surfaces_provenance_and_keeps_metadata_out_of_the_model_list( +def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every_status_surface( client, auth_as, monkeypatch, mock_prisma ): - """A real refetch through the reload route reports the file's stamp and the fetch ETag on every - status surface, while the ``_metadata`` block never shows up as a model anywhere.""" + """A real refetch through the reload route reports the git blob id of the exact bytes it fetched and + the fetch ETag on the reload response, the source route, and the schedule status alike.""" import httpx import litellm + from litellm.litellm_core_utils.get_model_cost_map import git_blob_id from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles _attach_litellm_config(mock_prisma) monkeypatch.setattr(ps, "prisma_client", mock_prisma) monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) - stamped = {**json.loads(_ROOT_COST_MAP.read_text()), "_metadata": {k: v for k, v in _PROVENANCE.items() if k != "etag"}} - served = httpx.Response(200, headers={"ETag": _PROVENANCE["etag"]}, content=json.dumps(stamped).encode()) + body = _ROOT_COST_MAP.read_bytes() + expected = {"source_revision": git_blob_id(body), "etag": _PROVENANCE["etag"]} + served = httpx.Response(200, headers={"ETag": _PROVENANCE["etag"]}, content=body) monkeypatch.setattr( "litellm.litellm_core_utils.get_model_cost_map._default_reload_client", lambda: httpx.AsyncClient(transport=httpx.MockTransport(lambda request: served)), @@ -144,18 +145,15 @@ def test_reload_model_cost_map_surfaces_provenance_and_keeps_metadata_out_of_the assert reload_response.status_code == 200 reload_body = reload_response.json() - assert {key: reload_body[key] for key in _PROVENANCE} == _PROVENANCE + assert {key: reload_body[key] for key in expected} == expected assert source_response.status_code == 200 source_body = source_response.json() - assert {key: source_body[key] for key in _PROVENANCE} == _PROVENANCE + assert {key: source_body[key] for key in expected} == expected assert source_body["source"] == "remote" assert status_response.status_code == 200 - assert {key: status_response.json()[key] for key in _PROVENANCE} == _PROVENANCE + assert {key: status_response.json()[key] for key in expected} == expected assert public_response.status_code == 200 - public_body = public_response.json() - assert "_metadata" not in public_body - assert "_metadata" not in litellm.model_cost - assert "gpt-4o" in public_body + assert "gpt-4o" in public_response.json() assert reload_body["models_count"] == len(litellm.model_cost) diff --git a/tests/test_litellm/test_auto_update_price_and_context_window_file.py b/tests/test_litellm/test_auto_update_price_and_context_window_file.py deleted file mode 100644 index d3cda09cd96..00000000000 --- a/tests/test_litellm/test_auto_update_price_and_context_window_file.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Tests for .github/scripts/auto_update_price_and_context_window_file.py.""" - -import importlib.util -import json -import re -import sys -from pathlib import Path -from typing import Final - -_REPO_ROOT: Final = Path(__file__).resolve().parents[2] -_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "auto_update_price_and_context_window_file.py" -_spec: Final = importlib.util.spec_from_file_location("auto_update_price_and_context_window_file", _MODULE_PATH) -script: Final = importlib.util.module_from_spec(_spec) -sys.modules[_spec.name] = script -_spec.loader.exec_module(script) - -_LOCAL_FILE: Final = "model_prices_and_context_window.json" -_GENERATED_AT: Final = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z") - - -def _openrouter_row(model_id: str) -> dict: - return {"id": model_id, "context_length": 8192, "pricing": {"prompt": "0.000001", "completion": "0.000002"}} - - -def _serve(openrouter_rows: list) -> object: - async def fetch_data(url: str) -> list: - return openrouter_rows if "openrouter" in url else [] - - return fetch_data - - -def _read_local(tmp_path: Path) -> dict: - return json.loads((tmp_path / _LOCAL_FILE).read_text()) - - -def test_main_stamps_provenance_only_when_the_sync_changed_the_file(tmp_path: Path, monkeypatch) -> None: - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("GITHUB_SHA", "feedface") - monkeypatch.setattr(script, "fetch_data", _serve([_openrouter_row("acme/x")])) - (tmp_path / _LOCAL_FILE).write_text(json.dumps({"sample_spec": {"input_cost_per_token": "USD"}}, indent=4) + "\n") - - script.main() - - written = _read_local(tmp_path) - assert written["openrouter/acme/x"]["litellm_provider"] == "openrouter" - assert written["_metadata"]["source_revision"] == "feedface" - assert _GENERATED_AT.fullmatch(written["_metadata"]["generated_at"]) - - sentinel = {**written, "_metadata": {**written["_metadata"], "generated_at": "2000-01-01T00:00:00Z"}} - (tmp_path / _LOCAL_FILE).write_text(json.dumps(sentinel, indent=4) + "\n") - - script.main() - - assert _read_local(tmp_path) == sentinel diff --git a/tests/test_litellm/test_cost_map_guard.py b/tests/test_litellm/test_cost_map_guard.py index 1a60cf81164..1b4330ed62c 100644 --- a/tests/test_litellm/test_cost_map_guard.py +++ b/tests/test_litellm/test_cost_map_guard.py @@ -141,26 +141,6 @@ def test_bot_may_not_change_special_root_keys() -> None: assert _failures(head) == ("bot PRs may not change fallback_generalizations",) -STAMP: Final = {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef0123456789abcdef01234567"} - - -def test_bot_may_stamp_and_restamp_metadata() -> None: - stamped = _snapshot({**BASE_MAP, "_metadata": STAMP}) - assert _failures(stamped) == () - assert _failures(stamped, bot=False) == () - - restamped = _snapshot( - { - **BASE_MAP, - "_metadata": {**STAMP, "generated_at": "2026-09-14T00:00:00Z"}, - "fallback_generalizations": {"rules": []}, - } - ) - assert guard.guard_failures(stamped, restamped, MAP_FILES, True) == ( - "bot PRs may not change fallback_generalizations", - ) - - def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str: text = _serialize(cost_map) (repo / guard.COST_MAP_PATH).write_text(text) diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index 3517f5840e8..c2c22c25998 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -98,25 +98,6 @@ def test_schema_rejects_malformed_entries(committed_schema: dict, entry: dict): assert not validator.is_valid({"some-model": entry}) -@pytest.mark.parametrize( - "metadata", - [ - "2026-09-07T00:00:00Z", - {"generated_at": "2026-09-07T00:00:00Z"}, - {"source_revision": "0123456789abcdef"}, - {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef", "author": "bot"}, - ], - ids=["not_an_object", "missing_revision", "missing_generated_at", "unknown_field"], -) -def test_schema_rejects_a_malformed_metadata_block(committed_schema: dict, metadata: object): - assert not build_validator(committed_schema).is_valid({"_metadata": metadata}) - - -def test_schema_accepts_the_provenance_stamp_as_a_non_model_root_key(committed_schema: dict): - stamp = {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef0123456789abcdef01234567"} - assert build_validator(committed_schema).is_valid({"_metadata": stamp}) - - def test_schema_accepts_minimal_and_unknown_optional_fields(committed_schema: dict): validator = build_validator(committed_schema) assert validator.is_valid({"some-model": {"litellm_provider": "openai"}}) diff --git a/tests/test_litellm/test_sync_together_ai_models.py b/tests/test_litellm/test_sync_together_ai_models.py index f58f573c208..b8a85bcfbdc 100644 --- a/tests/test_litellm/test_sync_together_ai_models.py +++ b/tests/test_litellm/test_sync_together_ai_models.py @@ -1,6 +1,5 @@ import importlib.util import json -import re from pathlib import Path from types import MappingProxyType @@ -370,58 +369,6 @@ def test_sync_is_idempotent_over_the_repo_cost_map() -> None: assert second.cost_map == first.cost_map -def test_stamp_metadata_adds_the_provenance_block_without_touching_models() -> None: - cost_map = {"sample_spec": {"input_cost_per_token": "USD"}, "together_ai/acme/x": {"mode": "chat"}} - - stamped = sync.stamp_metadata(cost_map, "2026-09-07T00:00:00Z", "feedface") - - assert stamped["_metadata"] == {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "feedface"} - assert {key: value for key, value in stamped.items() if key != "_metadata"} == cost_map - assert "_metadata" not in cost_map - - -def _write_registry(repo_root: Path, cost_map: dict) -> None: - for relpath in sync.COST_MAP_RELPATHS: - target = repo_root / relpath - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(json.dumps(cost_map, indent=4) + "\n") - - -def _read_registries(repo_root: Path) -> tuple[dict, ...]: - return tuple(json.loads((repo_root / relpath).read_text()) for relpath in sync.COST_MAP_RELPATHS) - - -def test_write_stamps_provenance_into_both_files_only_when_the_sync_changed_them(tmp_path: Path, monkeypatch) -> None: - monkeypatch.setenv("GITHUB_SHA", "feedface") - cost_map = json.loads((ROOT / "model_prices_and_context_window.json").read_text()) - dropped = next(f"together_ai/{model.id}" for model in RECORDED_CATALOG if f"together_ai/{model.id}" in cost_map) - _write_registry(tmp_path, {key: value for key, value in cost_map.items() if key not in {dropped, "_metadata"}}) - argv = ( - "--write", - "--models-json", - str(FIXTURES / "models_serverless.json"), - "--deprecations-md", - str(FIXTURES / "deprecations.md"), - "--repo-root", - str(tmp_path), - ) - - assert sync.main(argv) == 0 - - written = _read_registries(tmp_path) - assert written[0] == written[1] - assert dropped in written[0] - assert written[0]["_metadata"]["source_revision"] == "feedface" - assert re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", written[0]["_metadata"]["generated_at"]) - - sentinel = {**written[0], "_metadata": {**written[0]["_metadata"], "generated_at": "2000-01-01T00:00:00Z"}} - _write_registry(tmp_path, sentinel) - - assert sync.main(argv) == 0 - - assert _read_registries(tmp_path) == (sentinel, sentinel) - - def test_pr_body_lists_every_section_and_the_skipped_types() -> None: outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC) body = sync.render_pr_body(outcome) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index f6c9a4537a1..8a56a84ade7 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -21,7 +21,6 @@ from litellm._logging import ( verbose_logger, ) from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.get_model_cost_map import RESERVED_TOP_LEVEL_KEYS from litellm.proxy.utils import is_valid_api_key from litellm.types.utils import ( CallTypes, @@ -1219,12 +1218,15 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): with open(prod_json, "r") as model_prices_file: actual_json = json.load(model_prices_file) assert isinstance(actual_json, dict) - model_entries: Final = { - key: value for key, value in actual_json.items() if key not in RESERVED_TOP_LEVEL_KEYS - } + actual_json.pop( + "sample_spec", None + ) # remove the sample, whose schema is inconsistent with the real data + actual_json.pop( + "fallback_generalizations", None + ) # reserved meta key, not a model entry # Validate schema - validate(model_entries, INTENDED_SCHEMA) + validate(actual_json, INTENDED_SCHEMA) # Validate cost values # Define exceptions for models that are allowed to have costs > 1 @@ -1235,7 +1237,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "runwayml/seedance2", # 4K output is 150 credits/second = $1.50/second ] - is_valid, violations = validate_model_cost_values(model_entries, exceptions) + is_valid, violations = validate_model_cost_values(actual_json, exceptions) if not is_valid: error_message = "Cost validation failed:\n" + "\n".join(violations) @@ -1266,7 +1268,8 @@ def test_max_tokens_consistency(): inconsistencies = [] for model_name, config in models.items(): - if model_name in RESERVED_TOP_LEVEL_KEYS: + # Skip the sample_spec + if model_name == "sample_spec": continue # Check if both max_tokens and max_output_tokens exist diff --git a/ui/litellm-dashboard/src/components/price_data_reload.test.tsx b/ui/litellm-dashboard/src/components/price_data_reload.test.tsx index fde69675b72..101612993b0 100644 --- a/ui/litellm-dashboard/src/components/price_data_reload.test.tsx +++ b/ui/litellm-dashboard/src/components/price_data_reload.test.tsx @@ -33,15 +33,13 @@ const remoteSource = { is_env_forced: false, fallback_reason: null, loaded_at: null, - generated_at: null, source_revision: null, etag: null, model_count: 1234, }; const provenance = { loaded_at: "2026-09-07T10:00:00Z", - generated_at: "2026-09-06T23:38:47Z", - source_revision: "cd681a573fd9f5b6f15a1355f46178e4e9d374d2", + source_revision: "4273ec544726bf255ea920533e209e6022653bb4", etag: 'W/"eb8e9a53f4cc284b"', }; @@ -66,24 +64,22 @@ describe("PriceDataReload", () => { render(); expect(await screen.findByText("Source revision:")).toBeInTheDocument(); - expect(screen.getByText("cd681a573fd9")).toBeInTheDocument(); + expect(screen.getByText("4273ec544726")).toBeInTheDocument(); expect(screen.getByText("ETag:")).toBeInTheDocument(); expect(screen.getByText('W/"eb8e9a53f4cc284b"')).toBeInTheDocument(); - expect(screen.getByText("Generated at:")).toBeInTheDocument(); - expect(screen.getByText(new Date(provenance.generated_at).toLocaleString())).toBeInTheDocument(); expect(screen.getByText("Loaded at:")).toBeInTheDocument(); expect(screen.getByText(new Date(provenance.loaded_at).toLocaleString())).toBeInTheDocument(); }); - it("shows a malformed generated_at stamp as-is instead of Invalid Date", async () => { + it("shows a malformed loaded_at as-is instead of Invalid Date", async () => { vi.mocked(getModelCostMapSource).mockResolvedValue({ ...remoteSource, ...provenance, - generated_at: "yesterday-ish", + loaded_at: "yesterday-ish", } as never); render(); - expect(await screen.findByText("Generated at:")).toBeInTheDocument(); + expect(await screen.findByText("Loaded at:")).toBeInTheDocument(); expect(screen.getByText("yesterday-ish")).toBeInTheDocument(); expect(screen.queryByText("Invalid Date")).not.toBeInTheDocument(); }); @@ -92,7 +88,6 @@ describe("PriceDataReload", () => { render(); expect(await screen.findByText("Pricing Data Source")).toBeInTheDocument(); - expect(screen.queryByText("Generated at:")).not.toBeInTheDocument(); expect(screen.queryByText("Source revision:")).not.toBeInTheDocument(); expect(screen.queryByText("ETag:")).not.toBeInTheDocument(); expect(screen.queryByText("Loaded at:")).not.toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/price_data_reload.tsx b/ui/litellm-dashboard/src/components/price_data_reload.tsx index c152916f271..e5977a1b6e3 100644 --- a/ui/litellm-dashboard/src/components/price_data_reload.tsx +++ b/ui/litellm-dashboard/src/components/price_data_reload.tsx @@ -50,7 +50,6 @@ interface CostMapSourceInfo { is_env_forced: boolean; fallback_reason: string | null; loaded_at: string | null; - generated_at: string | null; source_revision: string | null; etag: string | null; model_count: number; @@ -105,13 +104,6 @@ const formatDateTime = (dateTimeString: string | null) => { const CostMapProvenanceRows: React.FC<{ sourceInfo: CostMapSourceInfo }> = ({ sourceInfo }) => ( <> - {sourceInfo.generated_at && ( -
- Generated at: - {formatDateTime(sourceInfo.generated_at)} -
- )} - {sourceInfo.source_revision && (
Source revision: diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7b9b24c9627..fc73d8264ef 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8685,7 +8685,7 @@ export interface paths { * - is_env_forced: true if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage * - fallback_reason: human-readable reason why remote failed (null on success) * - loaded_at: when this pod last loaded the map - * - generated_at, source_revision: the _metadata stamp inside the loaded file + * - source_revision: git blob id of the loaded file, what git rev-parse : prints for it * - etag: the ETag of the remote fetch (null for the bundled backup) * - model_count: number of models in the currently loaded cost map */ From bb52fd44fa425033313c7eac85bb5edaf92d71be Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 18:20:41 -0700 Subject: [PATCH 05/14] fix(cost_map): label the card's loaded_at as per-worker and cover the integrity-failure fallback --- .../test_get_model_cost_map.py | 20 +++++++++++++++++++ .../src/components/price_data_reload.test.tsx | 2 ++ .../src/components/price_data_reload.tsx | 9 +++++++++ 3 files changed, 31 insertions(+) diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index d9fe6d2f979..7c3ad283639 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -684,3 +684,23 @@ def test_boot_load_fallback_to_the_backup_reports_its_blob_id_and_drops_the_remo assert source["source"] == "local" assert source["etag"] is None assert source["source_revision"] == _bundled_blob_id() + + +def test_boot_load_that_fails_the_integrity_check_reports_the_backup_not_the_rejected_fetch(): + """A fetch that succeeds but fails integrity validation is thrown away, so the provenance must + describe the backup that got loaded, never the ETag or bytes of the map that was rejected.""" + remote, _ = _mock_client( + [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_real_map_bytes())], client_cls=httpx.Client + ) + get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=remote) + shrunk_body = b'{"gpt-5.4-mini": {"mode": "chat", "input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06}}' + shrunk, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"shrunk"'}, content=shrunk_body)], client_cls=httpx.Client) + + get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=shrunk) + + source = get_model_cost_map_source_info() + assert source["source"] == "local" + assert source["fallback_reason"] == "Remote data failed integrity validation" + assert source["etag"] is None + assert source["source_revision"] == _bundled_blob_id() + assert source["source_revision"] != git_blob_id(shrunk_body) diff --git a/ui/litellm-dashboard/src/components/price_data_reload.test.tsx b/ui/litellm-dashboard/src/components/price_data_reload.test.tsx index 101612993b0..3566ec2c3f2 100644 --- a/ui/litellm-dashboard/src/components/price_data_reload.test.tsx +++ b/ui/litellm-dashboard/src/components/price_data_reload.test.tsx @@ -68,6 +68,7 @@ describe("PriceDataReload", () => { expect(screen.getByText("ETag:")).toBeInTheDocument(); expect(screen.getByText('W/"eb8e9a53f4cc284b"')).toBeInTheDocument(); expect(screen.getByText("Loaded at:")).toBeInTheDocument(); + expect(screen.getByText(/worker that answered this request/)).toBeInTheDocument(); expect(screen.getByText(new Date(provenance.loaded_at).toLocaleString())).toBeInTheDocument(); }); @@ -91,6 +92,7 @@ describe("PriceDataReload", () => { expect(screen.queryByText("Source revision:")).not.toBeInTheDocument(); expect(screen.queryByText("ETag:")).not.toBeInTheDocument(); expect(screen.queryByText("Loaded at:")).not.toBeInTheDocument(); + expect(screen.queryByText(/worker that answered this request/)).not.toBeInTheDocument(); }); it("confirms an immediate reload and refreshes dependent data", async () => { diff --git a/ui/litellm-dashboard/src/components/price_data_reload.tsx b/ui/litellm-dashboard/src/components/price_data_reload.tsx index e5977a1b6e3..1363e306a31 100644 --- a/ui/litellm-dashboard/src/components/price_data_reload.tsx +++ b/ui/litellm-dashboard/src/components/price_data_reload.tsx @@ -132,6 +132,15 @@ const CostMapProvenanceRows: React.FC<{ sourceInfo: CostMapSourceInfo }> = ({ so {formatDateTime(sourceInfo.loaded_at)}
)} + + {sourceInfo.loaded_at && ( +
+ + + Reported by the worker that answered this request. Other workers pick up a reload on their next poll + +
+ )} ); From 0c6d4c539942f5f5ac2a723735d020f6ad91444f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:06:30 -0700 Subject: [PATCH 06/14] feat(cost_map): say on the card that Last run is deployment-wide while provenance is per worker --- ui/litellm-dashboard/src/components/price_data_reload.test.tsx | 1 + ui/litellm-dashboard/src/components/price_data_reload.tsx | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/price_data_reload.test.tsx b/ui/litellm-dashboard/src/components/price_data_reload.test.tsx index 3566ec2c3f2..4828d557053 100644 --- a/ui/litellm-dashboard/src/components/price_data_reload.test.tsx +++ b/ui/litellm-dashboard/src/components/price_data_reload.test.tsx @@ -69,6 +69,7 @@ describe("PriceDataReload", () => { expect(screen.getByText('W/"eb8e9a53f4cc284b"')).toBeInTheDocument(); expect(screen.getByText("Loaded at:")).toBeInTheDocument(); expect(screen.getByText(/worker that answered this request/)).toBeInTheDocument(); + expect(screen.getByText(/Last run time is the latest reload any worker recorded/)).toBeInTheDocument(); expect(screen.getByText(new Date(provenance.loaded_at).toLocaleString())).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/price_data_reload.tsx b/ui/litellm-dashboard/src/components/price_data_reload.tsx index 1363e306a31..bd2fb6721e0 100644 --- a/ui/litellm-dashboard/src/components/price_data_reload.tsx +++ b/ui/litellm-dashboard/src/components/price_data_reload.tsx @@ -137,7 +137,8 @@ const CostMapProvenanceRows: React.FC<{ sourceInfo: CostMapSourceInfo }> = ({ so
- Reported by the worker that answered this request. Other workers pick up a reload on their next poll + Reported by the worker that answered this request. Other workers pick up a reload on their next poll, and the + Last run time is the latest reload any worker recorded
)} From 41c0897c7aa21742968715d152b2948a4d0fb83f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:14:41 -0700 Subject: [PATCH 07/14] refactor(cost_map): drop docstrings from the provenance helpers and their tests --- litellm/litellm_core_utils/get_model_cost_map.py | 6 ------ .../litellm_core_utils/test_get_model_cost_map.py | 12 ------------ .../proxy/proxy_server/test_routes_model_cost_map.py | 3 --- 3 files changed, 21 deletions(-) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 2bdfbc66088..cdc4810ff04 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -45,7 +45,6 @@ def _count_model_entries(model_cost: dict) -> int: def git_blob_id(body: bytes) -> str: - """The sha1 git gives these bytes as a blob, so ``git rev-parse :`` reproduces it for the file""" return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest() @@ -70,7 +69,6 @@ class GetModelCostMap: @staticmethod def load_local_model_cost_map_with_revision() -> "ModelCostMapReloaded": - """The bundled backup map together with the git blob id of the file it was parsed from""" body: Final = GetModelCostMap.read_local_model_cost_map_bytes() content: Final = json.loads(body) return ModelCostMapReloaded(model_cost_map=content, revision=git_blob_id(body)) @@ -413,9 +411,6 @@ class CostMapSourceInfo(CostMapProvenance): def get_model_cost_map_provenance() -> CostMapProvenance: - """Which revision of the cost map this process serves: the git blob id of the bytes it loaded, the - same id ``git rev-parse :model_prices_and_context_window.json`` prints for a checkout, plus - the ETag the remote fetch returned (None for the bundled backup)""" return { "source_revision": _cost_map_source_info.source_revision, "etag": _cost_map_source_info.etag, @@ -520,7 +515,6 @@ def _finalize_model_cost_map(model_cost: dict) -> dict: def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded: - """Record which bytes this process now serves, then finalize the map they parsed into""" _cost_map_source_info.source_revision = loaded.revision _cost_map_source_info.etag = loaded.etag return replace(loaded, model_cost_map=_finalize_model_cost_map(loaded.model_cost_map)) diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 7c3ad283639..18794ea7eec 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -40,8 +40,6 @@ def _bundled_blob_id() -> str: def test_git_blob_id_is_what_git_hash_object_prints(): - """An operator checks a reported revision with ``git hash-object`` or ``git rev-parse :``, - so the id must be git's blob sha1 of the exact bytes, not a plain sha1 or a hash of the parsed JSON.""" assert git_blob_id(b'{"gpt-5.4-mini": {"mode": "chat"}}\n') == "18b9a8381e13a3b38a2128f184f631f95829e987" @@ -516,9 +514,6 @@ async def test_refetch_respects_local_env_override(monkeypatch): @pytest.mark.asyncio async def test_refetch_records_the_blob_id_of_the_bytes_served_and_the_fetch_etag(): - """A reload reports which revision of the map it swapped in: the git blob id of the exact bytes the - fetch returned, so ``git rev-parse :model_prices_and_context_window.json`` can confirm it, - plus the ETag the fetch returned.""" body = _real_map_bytes() client, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"abc123"'}, content=body)]) @@ -532,7 +527,6 @@ async def test_refetch_records_the_blob_id_of_the_bytes_served_and_the_fetch_eta @pytest.mark.asyncio async def test_refetch_revision_follows_the_bytes_not_the_url(): - """Two fetches of the same URL that return different bytes report different revisions.""" edited = json.loads(_real_map_bytes()) edited["gpt-5.4-mini"]["input_cost_per_token"] = 0.5 client, _ = _mock_client( @@ -549,8 +543,6 @@ async def test_refetch_revision_follows_the_bytes_not_the_url(): @pytest.mark.asyncio async def test_refetch_local_override_reports_the_bundled_blob_id_without_an_etag(monkeypatch): - """Forcing the bundled backup after a remote reload must report the backup's own blob id and drop the - remote ETag, since the map served is no longer the one that ETag identifies.""" remote, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"remote"'}, content=_real_map_bytes())]) await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=remote) monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -670,8 +662,6 @@ def test_boot_load_records_the_blob_id_of_the_bytes_served_and_the_fetch_etag(): def test_boot_load_fallback_to_the_backup_reports_its_blob_id_and_drops_the_remote_etag(): - """A boot that lands on the bundled backup reports the backup's own blob id and no ETag, even - when an earlier load in the same process had fetched the remote map.""" remote, _ = _mock_client( [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_real_map_bytes())], client_cls=httpx.Client ) @@ -687,8 +677,6 @@ def test_boot_load_fallback_to_the_backup_reports_its_blob_id_and_drops_the_remo def test_boot_load_that_fails_the_integrity_check_reports_the_backup_not_the_rejected_fetch(): - """A fetch that succeeds but fails integrity validation is thrown away, so the provenance must - describe the backup that got loaded, never the ETag or bytes of the map that was rejected.""" remote, _ = _mock_client( [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_real_map_bytes())], client_cls=httpx.Client ) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py index 0490993a314..36c364fb82b 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py @@ -50,7 +50,6 @@ def _attach_litellm_config(mock_prisma): def _pin_provenance(monkeypatch): - """Fix what this process reports as its cost map revision, independent of the map loaded at import.""" monkeypatch.setattr( "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map_provenance", lambda: dict(_PROVENANCE), @@ -110,8 +109,6 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every_status_surface( client, auth_as, monkeypatch, mock_prisma ): - """A real refetch through the reload route reports the git blob id of the exact bytes it fetched and - the fetch ETag on the reload response, the source route, and the schedule status alike.""" import httpx import litellm From 946d6f665ed84c59c4545d4519d9e029258046a2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:18:26 -0700 Subject: [PATCH 08/14] test(cost_map): assert runtime reloads refresh loaded_at --- .../test_get_model_cost_map.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 18794ea7eec..f72d175d579 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -335,6 +335,7 @@ def test_get_model_cost_map_stamps_loaded_at(monkeypatch): import functools import random +from datetime import datetime, timezone import httpx @@ -554,6 +555,27 @@ async def test_refetch_local_override_reports_the_bundled_blob_id_without_an_eta assert get_model_cost_map_provenance() == {"source_revision": _bundled_blob_id(), "etag": None} +@pytest.mark.asyncio +async def test_refetch_stamps_loaded_at_on_remote_and_local_reloads(monkeypatch): + from litellm.litellm_core_utils import get_model_cost_map as module + + client, _ = _mock_client([httpx.Response(200, content=_real_map_bytes())]) + monkeypatch.setattr(module._cost_map_source_info, "loaded_at", None) + before_remote = datetime.now(timezone.utc) + await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) + remote_loaded_at = module.get_model_cost_map_loaded_at() + assert remote_loaded_at is not None + assert before_remote <= remote_loaded_at <= datetime.now(timezone.utc) + + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(module._cost_map_source_info, "loaded_at", None) + before_local = datetime.now(timezone.utc) + await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0)) + local_loaded_at = module.get_model_cost_map_loaded_at() + assert local_loaded_at is not None + assert before_local <= local_loaded_at <= datetime.now(timezone.utc) + + # --------------------------------------------------------------------------- # get_model_cost_map: the boot-time load retries transient failures like a reload does # --------------------------------------------------------------------------- From 61ab4307ecfac49aad12feffcba265a21a270e6e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:47:31 -0700 Subject: [PATCH 09/14] test(cost_map): assert provenance without patching module state --- .../test_get_model_cost_map.py | 5 +-- .../test_routes_model_cost_map.py | 37 ++++++------------- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index f72d175d579..0495440c51c 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -310,14 +310,13 @@ def test_openrouter_catalog_costs_match_live_headline_rates(cost_map: dict): assert entry["output_cost_per_token"] != stale_out, model -def test_get_model_cost_map_stamps_loaded_at(monkeypatch): +def test_get_model_cost_map_stamps_loaded_at(): """The load time feeds each pod's reload-due decision; a load that does not stamp it would make manual reload requests race the proxy's startup""" from datetime import datetime, timezone from litellm.litellm_core_utils import get_model_cost_map as module - monkeypatch.setattr(module._cost_map_source_info, "loaded_at", None) client, _calls = _mock_client( [httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client ) @@ -560,7 +559,6 @@ async def test_refetch_stamps_loaded_at_on_remote_and_local_reloads(monkeypatch) from litellm.litellm_core_utils import get_model_cost_map as module client, _ = _mock_client([httpx.Response(200, content=_real_map_bytes())]) - monkeypatch.setattr(module._cost_map_source_info, "loaded_at", None) before_remote = datetime.now(timezone.utc) await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) remote_loaded_at = module.get_model_cost_map_loaded_at() @@ -568,7 +566,6 @@ async def test_refetch_stamps_loaded_at_on_remote_and_local_reloads(monkeypatch) assert before_remote <= remote_loaded_at <= datetime.now(timezone.utc) monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(module._cost_map_source_info, "loaded_at", None) before_local = datetime.now(timezone.utc) await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0)) local_loaded_at = module.get_model_cost_map_loaded_at() diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py index 36c364fb82b..40bd66ea91c 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_cost_map.py @@ -15,16 +15,15 @@ from pathlib import Path from unittest.mock import AsyncMock, MagicMock +from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map_provenance + from .conftest import VOLATILE_KEYS, normalize # Some response bodies include a "timestamp" — extend the volatile set so # dict-equality assertions remain stable. _VOLATILE = VOLATILE_KEYS | frozenset({"timestamp"}) -_PROVENANCE = { - "source_revision": "0123456789abcdef0123456789abcdef01234567", - "etag": 'W/"cost-map-etag"', -} +_SERVED_ETAG = 'W/"cost-map-etag"' _ROOT_COST_MAP = Path(__file__).resolve().parents[4] / "model_prices_and_context_window.json" @@ -49,13 +48,6 @@ def _attach_litellm_config(mock_prisma): return table -def _pin_provenance(monkeypatch): - monkeypatch.setattr( - "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map_provenance", - lambda: dict(_PROVENANCE), - ) - - # --------------------------------------------------------------------------- # POST /reload/model_cost_map # --------------------------------------------------------------------------- @@ -69,7 +61,6 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): table = _attach_litellm_config(mock_prisma) monkeypatch.setattr(ps, "prisma_client", mock_prisma) - _pin_provenance(monkeypatch) fake_cost_map = {"gpt-4": {"input_cost": 0.03}, "gpt-3.5": {"input_cost": 0.002}} monkeypatch.setattr( @@ -98,7 +89,7 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): "status": "success", "models_count": 2, "timestamp": "", - **_PROVENANCE, + **get_model_cost_map_provenance(), } assert table.upsert.await_count == 1 update_payload = table.upsert.await_args.kwargs["data"]["update"] @@ -120,8 +111,8 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every monkeypatch.setattr(ps, "prisma_client", mock_prisma) monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) body = _ROOT_COST_MAP.read_bytes() - expected = {"source_revision": git_blob_id(body), "etag": _PROVENANCE["etag"]} - served = httpx.Response(200, headers={"ETag": _PROVENANCE["etag"]}, content=body) + expected = {"source_revision": git_blob_id(body), "etag": _SERVED_ETAG} + served = httpx.Response(200, headers={"ETag": _SERVED_ETAG}, content=body) monkeypatch.setattr( "litellm.litellm_core_utils.get_model_cost_map._default_reload_client", lambda: httpx.AsyncClient(transport=httpx.MockTransport(lambda request: served)), @@ -339,7 +330,6 @@ def test_get_model_cost_map_reload_status_no_db_not_scheduled( from litellm.proxy._types import LitellmUserRoles monkeypatch.setattr(ps, "prisma_client", None) - _pin_provenance(monkeypatch) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/schedule/model_cost_map_reload/status") assert response.status_code == 200 @@ -348,7 +338,7 @@ def test_get_model_cost_map_reload_status_no_db_not_scheduled( "interval_hours": None, "last_run": None, "next_run": None, - **_PROVENANCE, + **get_model_cost_map_provenance(), } @@ -366,7 +356,6 @@ def test_get_model_cost_map_reload_status_scheduled( config_row.last_run_at = None table.find_unique = AsyncMock(return_value=config_row) monkeypatch.setattr(ps, "prisma_client", mock_prisma) - _pin_provenance(monkeypatch) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/schedule/model_cost_map_reload/status") @@ -376,7 +365,7 @@ def test_get_model_cost_map_reload_status_scheduled( "interval_hours": 12, "last_run": None, "next_run": None, - **_PROVENANCE, + **get_model_cost_map_provenance(), } @@ -396,7 +385,6 @@ def test_get_model_cost_map_reload_status_reports_persisted_last_run( config_row.last_run_at = datetime(2024, 1, 1, 6, 0, 0, tzinfo=timezone.utc) table.find_unique = AsyncMock(return_value=config_row) monkeypatch.setattr(ps, "prisma_client", mock_prisma) - _pin_provenance(monkeypatch) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/schedule/model_cost_map_reload/status") @@ -406,7 +394,7 @@ def test_get_model_cost_map_reload_status_reports_persisted_last_run( "interval_hours": 6, "last_run": "2024-01-01T06:00:00+00:00", "next_run": "2024-01-01T12:00:00+00:00", - **_PROVENANCE, + **get_model_cost_map_provenance(), } @@ -426,7 +414,6 @@ def test_get_model_cost_map_reload_status_no_config_not_scheduled( config_row.last_run_at = None table.find_unique = AsyncMock(return_value=config_row) monkeypatch.setattr(ps, "prisma_client", mock_prisma) - _pin_provenance(monkeypatch) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/schedule/model_cost_map_reload/status") @@ -436,7 +423,7 @@ def test_get_model_cost_map_reload_status_no_config_not_scheduled( "interval_hours": None, "last_run": None, "next_run": None, - **_PROVENANCE, + **get_model_cost_map_provenance(), } @@ -464,7 +451,7 @@ def test_get_model_cost_map_source_happy(client, auth_as, monkeypatch): "is_env_forced": False, "fallback_reason": None, "loaded_at": "2026-09-07T01:02:03+00:00", - **_PROVENANCE, + **get_model_cost_map_provenance(), } monkeypatch.setattr( "litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map_source_info", @@ -481,7 +468,7 @@ def test_get_model_cost_map_source_happy(client, auth_as, monkeypatch): "is_env_forced": False, "fallback_reason": None, "loaded_at": "2026-09-07T01:02:03+00:00", - **_PROVENANCE, + **get_model_cost_map_provenance(), "model_count": 3, } From 810d48f28fcdd0e2b29a64b5a238aa9697fae8e7 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 8 Sep 2026 17:40:12 -0700 Subject: [PATCH 10/14] chore(ci): extend diskcache scan exception to October 1 --- osv-scanner.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osv-scanner.toml b/osv-scanner.toml index 5b0339bdcd0..3e070fc8cf7 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -1,6 +1,6 @@ [[IgnoredVulns]] id = "GHSA-w8v5-vhqr-4h9v" -ignoreUntil = 2026-09-09 +ignoreUntil = 2026-10-01 reason = "diskcache has no fixed release published; remove this entry once one exists" [[IgnoredVulns]] From 43a1b2992adbb70e8fcef9c0ae431247d05743b3 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 01:00:26 +0000 Subject: [PATCH 11/14] fix(otel v2): restore the Datadog auth span and the last-wins callback merge (#40335) * fix(otel v2): restore the Datadog auth span and the last-wins callback merge Move @tracer.wrap() back onto user_api_key_auth so USE_DDTRACE=true emits the auth span again, and let a failure entry's callback_vars take part in the destination merge so the resolver picks the same account the runtime parser does Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel v2): drop docstrings from the two regression tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: rerun proxy-infra after the flaky test_check_migration process-tree test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/user_api_key_auth.py | 2 +- litellm/proxy/litellm_pre_call_utils.py | 9 +- .../otel/test_otel_v2_destinations.py | 29 +++- .../proxy/auth/test_user_api_key_auth.py | 128 ++++++++++++++++++ 4 files changed, 162 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 5a2a20f8f59..6b000489d5a 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2836,7 +2836,6 @@ async def _authorize_authenticated_request( return None -@tracer.wrap() def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth, request: Request | None = None) -> None: """Anchor the OTLP destinations this key or team overrides its traces to. @@ -2874,6 +2873,7 @@ def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth, request: Reque verbose_proxy_logger.debug("OTel V2: tenant destination resolution failed: %s", exc) +@tracer.wrap() async def user_api_key_auth( request: Request, api_key: str = fastapi.Security(api_key_header), diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d0eb75bc29a..924f84be5f4 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1041,7 +1041,9 @@ def resolve_tenant_otel_destinations( the request has an outcome, so honouring the filter would mean holding every span back until the call finishes. Those entries keep today's behaviour instead, where the tenant's credentials reach the backend through per-request tracer routing and - the operator's exporter is left alone. + the operator's exporter is left alone. Its ``callback_vars`` still take part in the + merge for a backend another entry made eligible, so the destination carries the + same credentials the runtime parser resolves for that request. A backend the request disabled dynamically, through the key's ``litellm_disabled_callbacks`` or the ``x-litellm-disable-callbacks`` header in @@ -1069,12 +1071,13 @@ def resolve_tenant_otel_destinations( callback for item in entries if (callback := _get_validated_callback_metadata(item=item, source="otel-destination")) is not None - if callback.callback_type != "failure" if callback.callback_name.lower() not in disabled ) return tuple( destination - for name in dict.fromkeys(callback.callback_name for callback in callbacks) + for name in dict.fromkeys( + callback.callback_name for callback in callbacks if callback.callback_type != "failure" + ) if ( destination := destination_for( name, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 1799381bada..67695d5aed8 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -2,7 +2,9 @@ import contextvars import time +from base64 import b64encode from collections.abc import Mapping +from functools import reduce from types import MappingProxyType import pytest @@ -49,8 +51,11 @@ from litellm.integrations.otel.presets.destinations import ( destination_for, ) from litellm.integrations.otel.presets.langfuse import langfuse_preset -from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.litellm_pre_call_utils import resolve_tenant_otel_destinations +from litellm.proxy._types import AddTeamCallback, UserAPIKeyAuth +from litellm.proxy.litellm_pre_call_utils import ( + convert_key_logging_metadata_to_callback, + resolve_tenant_otel_destinations, +) from litellm.types.utils import StandardCallbackDynamicParams LANGFUSE_DEST = OtelDestination( @@ -1822,6 +1827,26 @@ class TestTenantConfigAgreement: assert [d.endpoint for d in destinations] == ["http://key.local/api/public/otel"] + def test_a_failure_entry_still_wins_the_merge_next_to_a_success_entry(self): + entries = [ + {**self._entry("http://team.local"), "callback_type": "success"}, + { + **self._entry("http://key.local", langfuse_public_key="pk-failure", langfuse_secret_key="sk-failure"), + "callback_type": "failure", + }, + ] + runtime = reduce( + lambda merged, entry: convert_key_logging_metadata_to_callback(AddTeamCallback(**entry), merged), + entries, + None, + ) + + destinations = resolve_tenant_otel_destinations(UserAPIKeyAuth(team_metadata={"logging": entries})) + + assert runtime.callback_vars["langfuse_host"] == "http://key.local" + assert [d.endpoint for d in destinations] == ["http://key.local/api/public/otel"] + assert destinations[0].headers["Authorization"] == f"Basic {b64encode(b'pk-failure:sk-failure').decode()}" + @pytest.fixture def premium(self, monkeypatch): from litellm.proxy import proxy_server diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index f1a269cc00a..fd289e33ea6 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -1,8 +1,13 @@ import asyncio import json import logging +import os +import subprocess +import sys from contextlib import contextmanager from datetime import datetime, timedelta +from pathlib import Path +from textwrap import dedent from types import SimpleNamespace from unittest.mock import ANY, AsyncMock, MagicMock, patch @@ -7005,3 +7010,126 @@ class TestLitellmReceivedAtStamping: assert result == earlier assert request.state.litellm_received_at == earlier + + +_RECORDING_DDTRACE = dedent( + ''' + import functools + import inspect + + + class _Span: + def __enter__(self): + return self + + def __exit__(self, *exc): + return None + + + class _Tracer: + def __init__(self): + self.spans = [] + + def wrap(self, name=None, **kwargs): + def decorator(f): + span_name = name or f"{f.__module__}.{f.__name__}" + if inspect.iscoroutinefunction(f): + + @functools.wraps(f) + async def async_wrapped(*args, **kw): + self.spans.append(span_name) + return await f(*args, **kw) + + return async_wrapped + + @functools.wraps(f) + def wrapped(*args, **kw): + self.spans.append(span_name) + return f(*args, **kw) + + return wrapped + + return decorator + + def trace(self, name, **kwargs): + return _Span() + + def current_span(self): + return None + + def current_root_span(self): + return None + + + tracer = _Tracer() + ''' +) + +_DDTRACE_AUTH_PROBE = dedent( + ''' + import asyncio + import json + + import ddtrace + from fastapi import Request + from starlette.datastructures import URL + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import ProxyException + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + proxy_server.master_key = "sk-probe" + + + async def auth(api_key): + request = Request(scope={"type": "http", "headers": [], "method": "POST", "path": "/chat/completions"}) + request._url = URL(url="/chat/completions") + try: + await user_api_key_auth( + request=request, + api_key=api_key, + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + custom_litellm_key_header=None, + ) + return "accepted" + except ProxyException: + return "rejected" + + + async def main(): + outcomes = [await auth("Bearer sk-probe"), await auth("Bearer sk-wrong")] + print(json.dumps({"outcomes": outcomes, "spans": ddtrace.tracer.spans})) + + + asyncio.run(main()) + ''' +) + + +def test_user_api_key_auth_opens_a_datadog_span_for_accepted_and_rejected_keys(tmp_path: Path): + stub_root = tmp_path / "site" + (stub_root / "ddtrace").mkdir(parents=True) + (stub_root / "ddtrace" / "__init__.py").write_text(_RECORDING_DDTRACE) + probe = tmp_path / "probe.py" + probe.write_text(_DDTRACE_AUTH_PROBE) + repo_root = Path(litellm.__file__).resolve().parent.parent + env = { + **os.environ, + "USE_DDTRACE": "true", + "PYTHONPATH": os.pathsep.join( + [str(stub_root), str(repo_root)] + [p for p in (os.environ.get("PYTHONPATH"),) if p] + ), + } + + result = subprocess.run( + [sys.executable, str(probe)], env=env, cwd=repo_root, capture_output=True, text=True, check=False + ) + + assert result.returncode == 0, result.stderr[-4000:] + report = json.loads(result.stdout.strip().splitlines()[-1]) + assert report["outcomes"] == ["accepted", "rejected"] + auth_span = "litellm.proxy.auth.user_api_key_auth.user_api_key_auth" + assert [span for span in report["spans"] if span == auth_span] == [auth_span, auth_span] From 634852a18309a97f8cb24f18fabd48bb0b80c6a1 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 8 Sep 2026 18:00:01 -0700 Subject: [PATCH 12/14] fix(anthropic): key the /v1/messages prompt cache on Claude Code's session_id only The bridges derived prompt_cache_key as the first 64 chars of metadata.user_id. Claude Code packs a JSON object into that field whose prefix is the per-install device_id, so every session and subagent on one machine shared a single key, and a plain end-user id pinned all of that user's conversations to one slot. Parse the JSON and use session_id; send no key otherwise so the provider falls back to its own prompt-prefix hashing. An explicit prompt_cache_key still wins. Fixes #39145 --- .../experimental_pass_through/utils.py | 23 +++++- ...al_pass_through_adapters_transformation.py | 72 ++++++++++++++----- .../adapters/test_handler_prompt_cache_key.py | 12 ++-- .../test_responses_adapters_handler.py | 21 ++++-- .../test_responses_adapters_transformation.py | 33 ++++++--- 5 files changed, 123 insertions(+), 38 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/utils.py b/litellm/llms/anthropic/experimental_pass_through/utils.py index 55fe9c47faf..335a0e5641d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/utils.py @@ -3,6 +3,8 @@ from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final +from pydantic import BaseModel, ConfigDict, ValidationError + import litellm from litellm.types.utils import ModelInfo @@ -21,10 +23,27 @@ _EFFORT_DEGRADATION_CHAIN: Final[Mapping[str, tuple[str, ...]]] = MappingProxyTy _THINKING_OFF: Final = "none" +class _ClaudeCodeUserId(BaseModel): + """The JSON Claude Code packs into ``metadata.user_id``; only ``session_id`` is per conversation.""" + + model_config = ConfigDict(frozen=True) + + session_id: str + + def prompt_cache_key_from_user_id(user_id: object) -> str | None: - if user_id is None: + """The per-session key Claude Code carries inside ``metadata.user_id``, or nothing. + + Anthropic defines ``user_id`` as an opaque end-user id, so a plain string names a person, not + a conversation. Keying the provider cache on it pins every parallel session and subagent of that + person to one slot, which caches worse than the provider's own prompt-prefix hashing does. + """ + if not isinstance(user_id, str): + return None + try: + return _ClaudeCodeUserId.model_validate_json(user_id).session_id[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None + except ValidationError: return None - return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None def litellm_logging_obj_from_kwargs(kwargs: Mapping[str, object]) -> "LiteLLMLoggingObject | None": diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 30465ca25ba..c59ec70b015 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -1,4 +1,5 @@ import base64 +import json from typing import Any, Final, cast import pytest @@ -724,9 +725,14 @@ def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system(): ] -def _translate_with_metadata( - model: str, metadata: dict[str, str], custom_llm_provider: str | None -) -> dict[str, Any]: +def _claude_code_user_id(session_id: str) -> str: + return json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": session_id}) + + +CLAUDE_CODE_USER_ID: Final = _claude_code_user_id("session-abc") + + +def _translate_with_metadata(model: str, metadata: dict[str, str], custom_llm_provider: str | None) -> dict[str, Any]: openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( anthropic_message_request={ "model": model, @@ -739,23 +745,51 @@ def _translate_with_metadata( return cast(dict[str, Any], openai_request) -def test_translate_anthropic_to_openai_maps_user_id_to_prompt_cache_key_for_openai(): - openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": "session-abc"}, "openai") - assert openai_request["user"] == "session-abc" +def test_translate_anthropic_to_openai_maps_claude_code_session_id_to_prompt_cache_key_for_openai(): + openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": CLAUDE_CODE_USER_ID}, "openai") + assert openai_request["user"] == CLAUDE_CODE_USER_ID assert openai_request["prompt_cache_key"] == "session-abc" -def test_translate_anthropic_to_openai_truncates_prompt_cache_key_but_keeps_full_user(): - long_id = "".join(str(i % 10) for i in range(100)) - openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": long_id}, "openai") - assert openai_request["user"] == long_id - assert openai_request["prompt_cache_key"] == long_id[:64] - assert len(openai_request["prompt_cache_key"]) == 64 +def test_translate_anthropic_to_openai_gives_each_claude_code_session_its_own_prompt_cache_key(): + """BerriAI/litellm#39145: the first 64 chars of Claude Code's user_id are the per-install device_id.""" + keys = tuple( + _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": _claude_code_user_id(session_id)}, "openai")[ + "prompt_cache_key" + ] + for session_id in ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222") + ) + assert keys == ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222") + + +def test_translate_anthropic_to_openai_truncates_long_session_id_to_openai_limit(): + long_session_id = "".join(str(i % 10) for i in range(100)) + openai_request = _translate_with_metadata( + "openai/gpt-5.6-luna", {"user_id": _claude_code_user_id(long_session_id)}, "openai" + ) + assert openai_request["prompt_cache_key"] == long_session_id[:64] + + +@pytest.mark.parametrize( + "user_id", + [ + "alice", + "".join(str(i % 10) for i in range(100)), + json.dumps({"device_id": "d" * 64, "account_uuid": ""}), + json.dumps({"session_id": ""}), + json.dumps({"session_id": 123}), + "{not json", + ], +) +def test_translate_anthropic_to_openai_keeps_plain_user_id_off_prompt_cache_key(user_id: str): + openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": user_id}, "openai") + assert openai_request["user"] == user_id + assert "prompt_cache_key" not in openai_request @pytest.mark.parametrize("model", ["azure/my-gpt-5-deployment", "my-gpt-5-deployment"]) def test_translate_anthropic_to_openai_sets_prompt_cache_key_for_azure(model: str): - openai_request = _translate_with_metadata(model, {"user_id": "session-abc"}, "azure") + openai_request = _translate_with_metadata(model, {"user_id": CLAUDE_CODE_USER_ID}, "azure") assert openai_request["prompt_cache_key"] == "session-abc" @@ -772,8 +806,8 @@ def test_translate_anthropic_to_openai_sets_prompt_cache_key_for_azure(model: st def test_translate_anthropic_to_openai_skips_prompt_cache_key_when_provider_lacks_it( model: str, custom_llm_provider: str ): - openai_request = _translate_with_metadata(model, {"user_id": "session-abc"}, custom_llm_provider) - assert openai_request["user"] == "session-abc" + openai_request = _translate_with_metadata(model, {"user_id": CLAUDE_CODE_USER_ID}, custom_llm_provider) + assert openai_request["user"] == CLAUDE_CODE_USER_ID assert "prompt_cache_key" not in openai_request @@ -781,14 +815,14 @@ def test_translate_anthropic_to_openai_skips_prompt_cache_key_for_chained_litell assert "prompt_cache_key" in litellm.get_supported_openai_params( model="xai", custom_llm_provider="litellm_proxy" ) - openai_request = _translate_with_metadata("litellm_proxy/xai", {"user_id": "session-abc"}, "litellm_proxy") - assert openai_request["user"] == "session-abc" + openai_request = _translate_with_metadata("litellm_proxy/xai", {"user_id": CLAUDE_CODE_USER_ID}, "litellm_proxy") + assert openai_request["user"] == CLAUDE_CODE_USER_ID assert "prompt_cache_key" not in openai_request def test_translate_anthropic_to_openai_skips_prompt_cache_key_without_provider(): - openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": "session-abc"}, None) - assert openai_request["user"] == "session-abc" + openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": CLAUDE_CODE_USER_ID}, None) + assert openai_request["user"] == CLAUDE_CODE_USER_ID assert "prompt_cache_key" not in openai_request diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py index f48d51dbe1e..7dc7507120f 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_prompt_cache_key.py @@ -1,3 +1,4 @@ +import json import os import sys @@ -10,6 +11,7 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( ) MESSAGES = [{"role": "user", "content": "hello"}] +CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"}) def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, object] | None = None): @@ -17,7 +19,7 @@ def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, ob max_tokens=1024, messages=MESSAGES, model=model, - metadata={"user_id": "session-abc"}, + metadata={"user_id": CLAUDE_CODE_USER_ID}, thinking=thinking, extra_kwargs=extra_kwargs, ) @@ -26,7 +28,7 @@ def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, ob def test_prepare_completion_kwargs_derives_prompt_cache_key_for_openai_provider(): completion_kwargs = _prepare("openai/gpt-5.6-luna", {"custom_llm_provider": "openai"}) - assert completion_kwargs["user"] == "session-abc" + assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID assert completion_kwargs["prompt_cache_key"] == "session-abc" @@ -35,7 +37,7 @@ def test_prepare_completion_kwargs_prefers_explicit_prompt_cache_key_over_derive "openai/gpt-5.6-luna", {"custom_llm_provider": "openai", "prompt_cache_key": "explicit-key"}, ) - assert completion_kwargs["user"] == "session-abc" + assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID assert completion_kwargs["prompt_cache_key"] == "explicit-key" @@ -50,13 +52,13 @@ def test_prepare_completion_kwargs_skips_prompt_cache_key_without_provider_suppo model: str, extra_kwargs: dict[str, object] ): completion_kwargs = _prepare(model, extra_kwargs) - assert completion_kwargs["user"] == "session-abc" + assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID assert "prompt_cache_key" not in completion_kwargs def test_prepare_completion_kwargs_skips_prompt_cache_key_for_chained_litellm_proxy(): completion_kwargs = _prepare("litellm_proxy/xai", {"custom_llm_provider": "litellm_proxy"}) - assert completion_kwargs["user"] == "session-abc" + assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID assert "prompt_cache_key" not in completion_kwargs diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py index 3383813245a..16e8cf0e90e 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py @@ -16,6 +16,7 @@ from litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler ) MESSAGES = [{"role": "user", "content": "hello"}] +CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"}) RESPONSES_SSE_BODY = ( b"event: response.created\n" @@ -30,7 +31,19 @@ RESPONSES_SSE_BODY = ( ) -def test_build_responses_kwargs_derives_prompt_cache_key_from_user_id(): +def test_build_responses_kwargs_derives_prompt_cache_key_from_claude_code_session_id(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + metadata={"user_id": CLAUDE_CODE_USER_ID}, + extra_kwargs={"custom_llm_provider": "openai"}, + ) + assert responses_kwargs["user"] == CLAUDE_CODE_USER_ID[:64] + assert responses_kwargs["prompt_cache_key"] == "session-abc" + + +def test_build_responses_kwargs_sets_no_prompt_cache_key_for_plain_user_id(): responses_kwargs = _build_responses_kwargs( max_tokens=1024, messages=MESSAGES, @@ -39,7 +52,7 @@ def test_build_responses_kwargs_derives_prompt_cache_key_from_user_id(): extra_kwargs={"custom_llm_provider": "openai"}, ) assert responses_kwargs["user"] == "session-abc" - assert responses_kwargs["prompt_cache_key"] == "session-abc" + assert "prompt_cache_key" not in responses_kwargs def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived(): @@ -47,10 +60,10 @@ def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived() max_tokens=1024, messages=MESSAGES, model="openai/gpt-5.6-luna", - metadata={"user_id": "session-abc"}, + metadata={"user_id": CLAUDE_CODE_USER_ID}, extra_kwargs={"custom_llm_provider": "openai", "prompt_cache_key": "explicit-key"}, ) - assert responses_kwargs["user"] == "session-abc" + assert responses_kwargs["user"] == CLAUDE_CODE_USER_ID[:64] assert responses_kwargs["prompt_cache_key"] == "explicit-key" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 9f8414afa38..edcc7adddb7 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -1113,17 +1113,34 @@ class TestTranslateRequestBroaderCoverage: kwargs = _ADAPTER.translate_request(req) assert len(kwargs["user"]) == 64 - def test_metadata_user_id_mapped_to_prompt_cache_key(self): - req = _make_request(metadata={"user_id": "user-42"}) + def test_metadata_claude_code_session_id_mapped_to_prompt_cache_key(self): + user_id = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-42"}) + req = _make_request(metadata={"user_id": user_id}) kwargs = _ADAPTER.translate_request(req) - assert kwargs["prompt_cache_key"] == "user-42" + assert kwargs["user"] == user_id[:64] + assert kwargs["prompt_cache_key"] == "session-42" - def test_metadata_user_id_prompt_cache_key_truncated_to_first_64_chars(self): - long_id = "".join(str(i % 10) for i in range(100)) - req = _make_request(metadata={"user_id": long_id}) + def test_metadata_claude_code_sessions_get_distinct_prompt_cache_keys(self): + """BerriAI/litellm#39145: the first 64 chars of Claude Code's user_id are the per-install device_id.""" + keys = tuple( + _ADAPTER.translate_request( + _make_request( + metadata={"user_id": json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": sid})} + ) + )["prompt_cache_key"] + for sid in ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222") + ) + assert keys == ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222") + + @pytest.mark.parametrize( + "user_id", + ["user-42", "".join(str(i % 10) for i in range(100)), json.dumps({"device_id": "d" * 64}), "{not json"], + ) + def test_metadata_plain_user_id_sets_no_prompt_cache_key(self, user_id: str): + req = _make_request(metadata={"user_id": user_id}) kwargs = _ADAPTER.translate_request(req) - assert kwargs["prompt_cache_key"] == long_id[:64] - assert len(kwargs["prompt_cache_key"]) == 64 + assert kwargs["user"] == user_id[:64] + assert "prompt_cache_key" not in kwargs def test_metadata_empty_user_id_sets_no_prompt_cache_key(self): req = _make_request(metadata={"user_id": ""}) From 314e573529897752c882be9aac4985931bcc26bc Mon Sep 17 00:00:00 2001 From: tin-berri Date: Tue, 8 Sep 2026 18:28:46 -0700 Subject: [PATCH 13/14] feat(auto-router): refresh family reasoning presets (#40341) --- .../public_endpoints/autorouter_presets.json | 43 +++++++++++++++---- .../src/lib/autorouter_presets.test.ts | 17 ++++---- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/public_endpoints/autorouter_presets.json b/litellm/proxy/public_endpoints/autorouter_presets.json index 7d09db31127..7a251afc076 100644 --- a/litellm/proxy/public_endpoints/autorouter_presets.json +++ b/litellm/proxy/public_endpoints/autorouter_presets.json @@ -10,7 +10,12 @@ "REASONING": ["claude-opus-5"] }, "tier_model_configs": { - "REASONING": [{ "model_name": "claude-opus-5", "litellm_params": { "reasoning_effort": "high" } }] + "REASONING": [ + { + "model_name": "claude-opus-5", + "litellm_params": { "reasoning_effort": "high" } + } + ] }, "classifier_type": "heuristic_v2", "escalation_keywords": ["LITELLM ESCALATE"], @@ -23,16 +28,21 @@ }, "anthropic_family": { "label": "Anthropic Family", - "description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex, Opus at high thinking for reasoning.", + "description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex, Fable 5.1 at high thinking for reasoning.", "complexity_router_config": { "tiers": { "SIMPLE": ["claude-haiku-4-5"], "MEDIUM": ["claude-sonnet-5"], "COMPLEX": ["claude-opus-5"], - "REASONING": ["claude-opus-5"] + "REASONING": ["claude-fable-5-1"] }, "tier_model_configs": { - "REASONING": [{ "model_name": "claude-opus-5", "litellm_params": { "reasoning_effort": "high" } }] + "REASONING": [ + { + "model_name": "claude-fable-5-1", + "litellm_params": { "reasoning_effort": "high" } + } + ] }, "classifier_type": "heuristic", "escalation_keywords": ["LITELLM ESCALATE"], @@ -73,8 +83,18 @@ "REASONING": ["claude-opus-5"] }, "tier_model_configs": { - "MEDIUM": [{ "model_name": "muse-spark-1.2", "litellm_params": { "reasoning_effort": "xhigh" } }], - "COMPLEX": [{ "model_name": "kimi-k3", "litellm_params": { "reasoning_effort": "max" } }] + "MEDIUM": [ + { + "model_name": "muse-spark-1.2", + "litellm_params": { "reasoning_effort": "xhigh" } + } + ], + "COMPLEX": [ + { + "model_name": "kimi-k3", + "litellm_params": { "reasoning_effort": "max" } + } + ] }, "classifier_type": "llm", "classifier_llm_config": { @@ -93,16 +113,21 @@ }, "openai_family": { "label": "OpenAI Family", - "description": "Routes across the GPT model family: Luna for simple queries, Terra for medium, Sol for complex, Sol at xhigh thinking for reasoning.", + "description": "Routes across the GPT model family: Luna for simple queries, Terra for medium, Sol for complex, Astra at xhigh thinking for reasoning.", "complexity_router_config": { "tiers": { "SIMPLE": ["gpt-5.6-luna"], "MEDIUM": ["gpt-5.6-terra"], "COMPLEX": ["gpt-5.6-sol"], - "REASONING": ["gpt-5.6-sol"] + "REASONING": ["gpt-6-astra"] }, "tier_model_configs": { - "REASONING": [{ "model_name": "gpt-5.6-sol", "litellm_params": { "reasoning_effort": "xhigh" } }] + "REASONING": [ + { + "model_name": "gpt-6-astra", + "litellm_params": { "reasoning_effort": "xhigh" } + } + ] }, "classifier_type": "heuristic", "escalation_keywords": ["LITELLM ESCALATE"], diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index 5d1c7f73c71..8a5f83adbdf 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -149,13 +149,12 @@ describe("autorouter_presets", () => { ); }); - // Opus serves both tiers, so the effort is all that separates them and losing it fails silently. - it("pins the anthropic preset's reasoning tier to Opus at high thinking", () => { + it("pins the anthropic preset's reasoning tier to Fable 5.1 at high thinking", () => { const config = getPresetByKey("anthropic_family")!.complexity_router_config; expect(config.tiers.COMPLEX).toEqual(["claude-opus-5"]); - expect(config.tiers.REASONING).toEqual(["claude-opus-5"]); + expect(config.tiers.REASONING).toEqual(["claude-fable-5-1"]); expect(config.tier_model_configs).toEqual({ - REASONING: [{ model_name: "claude-opus-5", litellm_params: { reasoning_effort: "high" } }], + REASONING: [{ model_name: "claude-fable-5-1", litellm_params: { reasoning_effort: "high" } }], }); }); @@ -214,7 +213,7 @@ describe("autorouter_presets", () => { const preset = getPresetByKey("anthropic_family")!; const prefill = buildPresetPrefill(preset.complexity_router_config, groupsOnly(getRequiredModelsInPreset(preset))); expect(prefill.complexityRouterConfig.tier_model_params).toEqual({ - REASONING: { "claude-opus-5": { reasoning_effort: "high" } }, + REASONING: { "claude-fable-5-1": { reasoning_effort: "high" } }, }); }); @@ -227,21 +226,21 @@ describe("autorouter_presets", () => { }); }); - it("pins the OpenAI preset to the Luna, Terra, and Sol progression", () => { + it("pins the OpenAI preset to the Luna, Terra, Sol, and Astra progression", () => { const preset = getPresetByKey("openai_family")!; const expectedTiers = { SIMPLE: ["gpt-5.6-luna"], MEDIUM: ["gpt-5.6-terra"], COMPLEX: ["gpt-5.6-sol"], - REASONING: ["gpt-5.6-sol"], + REASONING: ["gpt-6-astra"], }; expect(preset.complexity_router_config.tiers).toEqual(expectedTiers); expect(preset.complexity_router_config.tier_model_configs).toEqual({ - REASONING: [{ model_name: "gpt-5.6-sol", litellm_params: { reasoning_effort: "xhigh" } }], + REASONING: [{ model_name: "gpt-6-astra", litellm_params: { reasoning_effort: "xhigh" } }], }); const prefill = buildPresetPrefill(preset.complexity_router_config, groupsOnly(getRequiredModelsInPreset(preset))); expect(prefill.complexityRouterConfig.tier_model_params).toEqual({ - REASONING: { "gpt-5.6-sol": { reasoning_effort: "xhigh" } }, + REASONING: { "gpt-6-astra": { reasoning_effort: "xhigh" } }, }); }); From fb21852f7bc5ba2a1d79a89b5d46a1ec6fc1e56d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 8 Sep 2026 18:35:25 -0700 Subject: [PATCH 14/14] test(mcp): resolve current manager in proxy fixtures --- .../proxy/_experimental/mcp_server/conftest.py | 8 ++++---- .../proxy/_experimental/mcp_server/test_mcp_proxy_mode.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/conftest.py b/tests/test_litellm/proxy/_experimental/mcp_server/conftest.py index 2ccba2b2055..9a66f130d24 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/conftest.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/conftest.py @@ -2,15 +2,15 @@ import os import pytest -from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, -) - @pytest.fixture(autouse=True) def _hermetic_mcp_server_registry(): """Restore the singleton ``global_mcp_server_manager``'s registry state around every test, so entries seeded by one test never leak into another on a shared shard.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + saved_registry = dict(global_mcp_server_manager.registry) saved_config_servers = dict(global_mcp_server_manager.config_mcp_servers) saved_tool_mapping = dict(global_mcp_server_manager.tool_name_to_mcp_server_name_mapping) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py index e86fc27fed7..413785529d5 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py @@ -7,7 +7,6 @@ from mcp.types import CallToolResult, TextContent, Tool from litellm.proxy._experimental.mcp_server import server from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing -from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager from litellm.proxy._experimental.mcp_server.tool_search import ( MCP_PROXY_CALL_TOOL_NAME, MCP_PROXY_SCHEMA_TOOL_NAME, @@ -271,6 +270,8 @@ class TestMcpProxyAuthorizationScope: @pytest.fixture def rig(self) -> Iterator[AsyncMock]: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + upstream = { "srv-alpha": [_upstream_tool("alpha", "add"), _upstream_tool("alpha", "multiply")], "srv-beta": [_upstream_tool("beta", "add")],