mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
feat(cost_map): stamp and surface generated_at and source revision provenance
The cost map JSON now carries a top-level `_metadata` block with `generated_at` and `source_revision`, written by the two bot writers only when model data changed. The loader pops it before the map becomes `litellm.model_cost`, records it next to the fetch ETag, and `/reload/model_cost_map`, `/model/cost_map/source`, and the reload schedule status return it. The Price Data Reload card shows the stamp, the ETag, and when the pod loaded the map. The schema and the cost map guard treat `_metadata` as a non-model root key
This commit is contained in:
mateo-berri
parent
cd681a573f
commit
0710231acc
18 changed files with 636 additions and 28 deletions
|
|
@ -1,6 +1,9 @@
|
|||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# Asynchronously fetch data from a given URL
|
||||
async def fetch_data(url):
|
||||
|
|
@ -31,13 +34,28 @@ def sync_local_data_with_remote(local_data, remote_data):
|
|||
for key in (set(remote_data) - set(local_data)):
|
||||
local_data[key] = remote_data[key]
|
||||
|
||||
def utc_now_iso():
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def source_revision():
|
||||
from_env = os.environ.get("GITHUB_SHA")
|
||||
if from_env:
|
||||
return from_env
|
||||
return subprocess.run(["git", "rev-parse", "HEAD"], check=True, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def stamp_metadata(data, generated_at, revision):
|
||||
return {**data, "_metadata": {"generated_at": generated_at, "source_revision": revision}}
|
||||
|
||||
|
||||
# Write data to the json file
|
||||
def write_to_file(file_path, data):
|
||||
try:
|
||||
# Open the file in write mode
|
||||
with open(file_path, "w") as file:
|
||||
# Dump the data as JSON into the file
|
||||
json.dump(data, file, indent=4)
|
||||
file.write(json.dumps(data, indent=4) + "\n")
|
||||
print("Values updated successfully.")
|
||||
except Exception as e:
|
||||
# Print an error message if writing to file fails
|
||||
|
|
@ -149,8 +167,13 @@ def main():
|
|||
|
||||
# If both local and openrouter data are available, synchronize and save
|
||||
if local_data and all_remote_data:
|
||||
before = json.dumps(local_data, sort_keys=True)
|
||||
sync_local_data_with_remote(local_data, all_remote_data)
|
||||
write_to_file(local_file_path, local_data)
|
||||
changed = json.dumps(local_data, sort_keys=True) != before
|
||||
write_to_file(
|
||||
local_file_path,
|
||||
stamp_metadata(local_data, utc_now_iso(), source_revision()) if changed else local_data,
|
||||
)
|
||||
else:
|
||||
print("Failed to fetch model data from either local file or URL.")
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@
|
|||
|
||||
Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file,
|
||||
and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named
|
||||
litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models.
|
||||
litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models, plus
|
||||
restamp the _metadata provenance block.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -15,7 +16,7 @@ from collections.abc import Sequence
|
|||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
|
||||
from generate_model_prices_schema import BOT_LOCKED_ROOT_KEYS, build_schema, render, validation_errors
|
||||
|
||||
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
|
||||
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
|
||||
|
|
@ -102,7 +103,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
|
|||
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),
|
||||
*(
|
||||
f"bot PRs may not change {key}"
|
||||
for key in sorted(SPECIAL_ROOT_KEYS)
|
||||
for key in sorted(BOT_LOCKED_ROOT_KEYS)
|
||||
if base_map.get(key) != head_map.get(key)
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -11,7 +11,9 @@ REPO_ROOT = Path(__file__).parent.parent
|
|||
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
|
||||
SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
|
||||
|
||||
SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"})
|
||||
METADATA_KEY = "_metadata"
|
||||
SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations", METADATA_KEY})
|
||||
BOT_LOCKED_ROOT_KEYS = SPECIAL_ROOT_KEYS - {METADATA_KEY}
|
||||
|
||||
JsonSchema = dict
|
||||
|
||||
|
|
@ -271,13 +273,26 @@ def build_schema(prices: dict) -> JsonSchema:
|
|||
"description": (
|
||||
"Schema for LiteLLM's model price and context window registry "
|
||||
"(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). "
|
||||
"Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, "
|
||||
"Every top-level key except '_metadata', 'sample_spec', and 'fallback_generalizations' is a model id, "
|
||||
"optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. "
|
||||
"All costs are USD per unit. New optional fields are added regularly, so consumers should "
|
||||
"ignore unknown fields rather than reject them."
|
||||
),
|
||||
"type": "object",
|
||||
"properties": {
|
||||
METADATA_KEY: {
|
||||
"type": "object",
|
||||
"description": (
|
||||
"Provenance of this file: when an automated sync last regenerated it and the commit it "
|
||||
"ran against. Human edits leave it untouched; not a model entry."
|
||||
),
|
||||
"properties": {
|
||||
"generated_at": {"type": "string", "format": "date-time"},
|
||||
"source_revision": STRING,
|
||||
},
|
||||
"required": ["generated_at", "source_revision"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"sample_spec": {
|
||||
"type": "object",
|
||||
"description": (
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ from importlib.resources import files
|
|||
from typing import Final, Protocol
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm.constants import (
|
||||
|
|
@ -31,10 +33,11 @@ from litellm.litellm_core_utils.fallback_generalizations import (
|
|||
)
|
||||
|
||||
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
|
||||
METADATA_KEY: Final = "_metadata"
|
||||
|
||||
# Reserved top-level keys that are not model entries. They must be excluded
|
||||
# from the model-count integrity check so a real upstream shrink can't be masked.
|
||||
RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY})
|
||||
RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY, METADATA_KEY})
|
||||
|
||||
|
||||
def _count_model_entries(model_cost: dict) -> int:
|
||||
|
|
@ -166,6 +169,7 @@ MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class ModelCostMapReloaded:
|
||||
model_cost_map: dict # mutable-ok: adopted as litellm.model_cost, whose consumer contract is a plain mutable dict
|
||||
etag: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -254,7 +258,7 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp
|
|||
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
|
||||
if not isinstance(parsed, dict):
|
||||
return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
|
||||
return ModelCostMapReloaded(model_cost_map=parsed)
|
||||
return ModelCostMapReloaded(model_cost_map=parsed, etag=response.headers.get("etag"))
|
||||
|
||||
|
||||
def _next_retry_wait(
|
||||
|
|
@ -328,10 +332,12 @@ async def refetch_model_cost_map(
|
|||
map they already have.
|
||||
"""
|
||||
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.url = None
|
||||
_cost_map_source_info.is_env_forced = True
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
_cost_map_source_info.etag = None
|
||||
return ModelCostMapReloaded(
|
||||
model_cost_map=_finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map())
|
||||
)
|
||||
|
|
@ -355,11 +361,13 @@ async def refetch_model_cost_map(
|
|||
backup_model_count=GetModelCostMap._get_backup_model_count(),
|
||||
):
|
||||
return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} failed integrity validation")
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
_cost_map_source_info.source = "remote"
|
||||
_cost_map_source_info.url = url
|
||||
_cost_map_source_info.is_env_forced = False
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return ModelCostMapReloaded(model_cost_map=_finalize_model_cost_map(result.model_cost_map))
|
||||
_cost_map_source_info.etag = result.etag
|
||||
return ModelCostMapReloaded(model_cost_map=_finalize_model_cost_map(result.model_cost_map), etag=result.etag)
|
||||
|
||||
|
||||
class ModelCostMapSourceInfo:
|
||||
|
|
@ -370,13 +378,60 @@ class ModelCostMapSourceInfo:
|
|||
is_env_forced: bool = False
|
||||
fallback_reason: str | None = None
|
||||
loaded_at: "datetime | None" = None
|
||||
generated_at: str | None = None
|
||||
source_revision: str | None = None
|
||||
etag: str | None = None
|
||||
|
||||
|
||||
# Module-level singleton tracking the source of the current cost map
|
||||
_cost_map_source_info: Final = ModelCostMapSourceInfo()
|
||||
|
||||
|
||||
def get_model_cost_map_source_info() -> dict:
|
||||
class CostMapMetadata(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore")
|
||||
|
||||
generated_at: str | None = None
|
||||
source_revision: str | None = None
|
||||
|
||||
|
||||
_EMPTY_METADATA: Final = CostMapMetadata()
|
||||
|
||||
|
||||
def _parse_metadata(raw: object) -> CostMapMetadata:
|
||||
if raw is None:
|
||||
return _EMPTY_METADATA
|
||||
try:
|
||||
return CostMapMetadata.model_validate(raw)
|
||||
except ValidationError as error:
|
||||
verbose_logger.warning("LiteLLM: ignoring a malformed %s block in the model cost map: %s", METADATA_KEY, error)
|
||||
return _EMPTY_METADATA
|
||||
|
||||
|
||||
class CostMapProvenance(TypedDict):
|
||||
generated_at: ReadOnly[str | None]
|
||||
source_revision: ReadOnly[str | None]
|
||||
etag: ReadOnly[str | None]
|
||||
|
||||
|
||||
class CostMapSourceInfo(CostMapProvenance):
|
||||
source: ReadOnly[str]
|
||||
url: ReadOnly[str | None]
|
||||
is_env_forced: ReadOnly[bool]
|
||||
fallback_reason: ReadOnly[str | None]
|
||||
loaded_at: ReadOnly[str | None]
|
||||
|
||||
|
||||
def get_model_cost_map_provenance() -> CostMapProvenance:
|
||||
"""Which revision of the cost map this process serves: the ``_metadata`` stamp the file
|
||||
carries plus the ETag the remote fetch returned (None for the bundled backup)"""
|
||||
return {
|
||||
"generated_at": _cost_map_source_info.generated_at,
|
||||
"source_revision": _cost_map_source_info.source_revision,
|
||||
"etag": _cost_map_source_info.etag,
|
||||
}
|
||||
|
||||
|
||||
def get_model_cost_map_source_info() -> CostMapSourceInfo:
|
||||
"""
|
||||
Return metadata about where the current model cost map was loaded from.
|
||||
|
||||
|
|
@ -385,12 +440,20 @@ def get_model_cost_map_source_info() -> dict:
|
|||
- url: the remote URL attempted (or None for local-only)
|
||||
- is_env_forced: True if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage
|
||||
- fallback_reason: human-readable reason if remote failed and local was used
|
||||
- loaded_at: ISO 8601 time this process last loaded the map
|
||||
- generated_at, source_revision: the ``_metadata`` stamp inside the loaded file
|
||||
- etag: the ETag of the remote fetch (None for the bundled backup)
|
||||
"""
|
||||
loaded_at: Final = _cost_map_source_info.loaded_at
|
||||
return {
|
||||
"source": _cost_map_source_info.source,
|
||||
"url": _cost_map_source_info.url,
|
||||
"is_env_forced": _cost_map_source_info.is_env_forced,
|
||||
"fallback_reason": _cost_map_source_info.fallback_reason,
|
||||
"loaded_at": loaded_at.isoformat() if loaded_at is not None else None,
|
||||
"generated_at": _cost_map_source_info.generated_at,
|
||||
"source_revision": _cost_map_source_info.source_revision,
|
||||
"etag": _cost_map_source_info.etag,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -455,14 +518,18 @@ def _expand_model_aliases(model_cost: dict) -> dict:
|
|||
|
||||
|
||||
def _finalize_model_cost_map(model_cost: dict) -> dict:
|
||||
"""Extract fallback generalizations out of the raw map, then expand aliases.
|
||||
"""Extract fallback generalizations and the provenance stamp out of the raw map, then expand aliases.
|
||||
|
||||
The ``fallback_generalizations`` block is installed into the generalizations
|
||||
module and removed from the map so it is never treated as a model entry.
|
||||
module and the ``_metadata`` block into the source info; both are removed from
|
||||
the map so neither is ever treated as a model entry.
|
||||
"""
|
||||
raw: Final = model_cost.pop(FALLBACK_GENERALIZATIONS_KEY, None)
|
||||
rules: Final = raw.get("rules") if isinstance(raw, dict) else None
|
||||
set_fallback_generalizations(rules)
|
||||
metadata: Final = _parse_metadata(model_cost.pop(METADATA_KEY, None))
|
||||
_cost_map_source_info.generated_at = metadata.generated_at
|
||||
_cost_map_source_info.source_revision = metadata.source_revision
|
||||
return _expand_model_aliases(model_cost)
|
||||
|
||||
|
||||
|
|
@ -494,10 +561,12 @@ def get_model_cost_map(
|
|||
_cost_map_source_info.url = None
|
||||
_cost_map_source_info.is_env_forced = True
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
_cost_map_source_info.etag = None
|
||||
return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map())
|
||||
|
||||
_cost_map_source_info.url = url
|
||||
_cost_map_source_info.is_env_forced = False
|
||||
_cost_map_source_info.etag = None
|
||||
|
||||
result: Final = _fetch_remote_model_cost_map_with_retry_sync(
|
||||
url=url,
|
||||
|
|
@ -533,4 +602,5 @@ def get_model_cost_map(
|
|||
|
||||
_cost_map_source_info.source = "remote"
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
_cost_map_source_info.etag = result.etag
|
||||
return _finalize_model_cost_map(content)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
{
|
||||
"_metadata": {
|
||||
"generated_at": "2026-09-07T23:38:47Z",
|
||||
"source_revision": "cd681a573fd9f5b6f15a1355f46178e4e9d374d2"
|
||||
},
|
||||
"sample_spec": {
|
||||
"code_interpreter_cost_per_session": 0.0,
|
||||
"computer_use_input_cost_per_1k_tokens": 0.0,
|
||||
|
|
|
|||
|
|
@ -17749,6 +17749,7 @@ async def reload_model_cost_map(
|
|||
# Immediately reload the model cost map in the current pod
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
ModelCostMapReloadUnavailable,
|
||||
get_model_cost_map_provenance,
|
||||
refetch_model_cost_map,
|
||||
)
|
||||
|
||||
|
|
@ -17762,6 +17763,7 @@ async def reload_model_cost_map(
|
|||
models_count = _swap_in_model_cost_map(reload_result.model_cost_map)
|
||||
current_time = utc_now()
|
||||
proxy_config.model_cost_map_loaded_at = current_time
|
||||
provenance: Final = get_model_cost_map_provenance()
|
||||
|
||||
# Publish a new revision so every other pod reloads on its next poll; this pod has
|
||||
# already served it, so adopt it here rather than reloading again a tick later
|
||||
|
|
@ -17776,6 +17778,7 @@ async def reload_model_cost_map(
|
|||
"status": "success",
|
||||
"models_count": models_count,
|
||||
"timestamp": current_time.isoformat(),
|
||||
**provenance,
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
|
|
@ -17896,12 +17899,17 @@ async def get_model_cost_map_reload_status(
|
|||
|
||||
try:
|
||||
global prisma_client
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
get_model_cost_map_provenance,
|
||||
)
|
||||
|
||||
provenance: Final = get_model_cost_map_provenance()
|
||||
if prisma_client is None:
|
||||
verbose_proxy_logger.info("No database connection, returning not scheduled")
|
||||
return reload_schedule_status(None)
|
||||
return {**reload_schedule_status(None), **provenance}
|
||||
|
||||
return reload_schedule_status(await read_reload_schedule(prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME))
|
||||
schedule: Final = await read_reload_schedule(prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME)
|
||||
return {**reload_schedule_status(schedule), **provenance}
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Failed to get model cost map reload status: %s", e)
|
||||
raise HTTPException(
|
||||
|
|
@ -17929,6 +17937,9 @@ async def get_model_cost_map_source(
|
|||
- url: the remote URL that was attempted (null when env-forced local)
|
||||
- is_env_forced: true if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage
|
||||
- fallback_reason: human-readable reason why remote failed (null on success)
|
||||
- loaded_at: when this pod last loaded the map
|
||||
- generated_at, source_revision: the _metadata stamp inside the loaded file
|
||||
- etag: the ETag of the remote fetch (null for the bundled backup)
|
||||
- model_count: number of models in the currently loaded cost map
|
||||
"""
|
||||
# Read-only source info — admin viewers can read.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
{
|
||||
"_metadata": {
|
||||
"generated_at": "2026-09-07T23:38:47Z",
|
||||
"source_revision": "cd681a573fd9f5b6f15a1355f46178e4e9d374d2"
|
||||
},
|
||||
"sample_spec": {
|
||||
"code_interpreter_cost_per_session": 0.0,
|
||||
"computer_use_input_cost_per_1k_tokens": 0.0,
|
||||
|
|
|
|||
|
|
@ -1,9 +1,27 @@
|
|||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "LiteLLM model_prices_and_context_window.json",
|
||||
"description": "Schema for LiteLLM's model price and context window registry (https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. All costs are USD per unit. New optional fields are added regularly, so consumers should ignore unknown fields rather than reject them.",
|
||||
"description": "Schema for LiteLLM's model price and context window registry (https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Every top-level key except '_metadata', 'sample_spec', and 'fallback_generalizations' is a model id, optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. All costs are USD per unit. New optional fields are added regularly, so consumers should ignore unknown fields rather than reject them.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"_metadata": {
|
||||
"type": "object",
|
||||
"description": "Provenance of this file: when an automated sync last regenerated it and the commit it ran against. Human edits leave it untouched; not a model entry.",
|
||||
"properties": {
|
||||
"generated_at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"source_revision": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"generated_at",
|
||||
"source_revision"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"sample_spec": {
|
||||
"type": "object",
|
||||
"description": "Documentation placeholder illustrating the entry shape; not a real model and not schema-conformant (several values are prose)."
|
||||
|
|
|
|||
|
|
@ -19,9 +19,11 @@ import argparse
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
|
@ -33,6 +35,7 @@ MODELS_URL: Final = "https://api.together.ai/v1/models?serverless"
|
|||
DEPRECATIONS_URL: Final = "https://docs.together.ai/docs/deprecations.md"
|
||||
PROVIDER: Final = "together_ai"
|
||||
PREFIX: Final = "together_ai/"
|
||||
METADATA_KEY: Final = "_metadata"
|
||||
SOURCE_URL: Final = "https://docs.together.ai/docs/serverless-models"
|
||||
COST_MAP_RELPATHS: Final = (
|
||||
"model_prices_and_context_window.json",
|
||||
|
|
@ -495,6 +498,23 @@ def _serialize(cost_map: CostMap) -> str:
|
|||
return json.dumps(cost_map, indent=4, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def stamp_metadata(cost_map: CostMap, generated_at: str, source_revision: str) -> CostMap:
|
||||
return {**cost_map, METADATA_KEY: {"generated_at": generated_at, "source_revision": source_revision}}
|
||||
|
||||
|
||||
def _utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _source_revision(repo_root: Path) -> str:
|
||||
from_env: Final = os.environ.get("GITHUB_SHA")
|
||||
if from_env:
|
||||
return from_env
|
||||
return subprocess.run(
|
||||
("git", "rev-parse", "HEAD"), cwd=repo_root, check=True, capture_output=True, text=True
|
||||
).stdout.strip()
|
||||
|
||||
|
||||
def main(argv: Sequence[str]) -> int:
|
||||
parser: Final = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--write", action="store_true", help="apply the sync to the cost map files (default: dry run)")
|
||||
|
|
@ -527,8 +547,9 @@ def main(argv: Sequence[str]) -> int:
|
|||
if args.pr_body_file is not None:
|
||||
args.pr_body_file.write_text(body)
|
||||
if args.write and outcome.has_changes:
|
||||
stamped: Final = _serialize(stamp_metadata(outcome.cost_map, _utc_now_iso(), _source_revision(args.repo_root)))
|
||||
for relpath in COST_MAP_RELPATHS:
|
||||
(args.repo_root / relpath).write_text(_serialize(outcome.cost_map))
|
||||
(args.repo_root / relpath).write_text(stamped)
|
||||
print(render_summary(outcome))
|
||||
print()
|
||||
print(body)
|
||||
|
|
|
|||
|
|
@ -17,9 +17,11 @@ from litellm.litellm_core_utils.fallback_generalizations import (
|
|||
)
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
FALLBACK_GENERALIZATIONS_KEY,
|
||||
METADATA_KEY,
|
||||
GetModelCostMap,
|
||||
_count_model_entries,
|
||||
_finalize_model_cost_map,
|
||||
get_model_cost_map_provenance,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -31,6 +33,20 @@ def _load_root_cost_map() -> dict:
|
|||
return json.load(f)
|
||||
|
||||
|
||||
def _load_bundled_stamp() -> dict:
|
||||
path = os.path.join(
|
||||
os.path.dirname(__file__), "../../../litellm/model_prices_and_context_window_backup.json"
|
||||
)
|
||||
with open(path) as f:
|
||||
return json.load(f)[METADATA_KEY]
|
||||
|
||||
|
||||
_STAMP = {
|
||||
"generated_at": "2026-09-07T00:00:00Z",
|
||||
"source_revision": "0123456789abcdef0123456789abcdef01234567",
|
||||
}
|
||||
|
||||
|
||||
def _make_models(n: int) -> dict:
|
||||
return {
|
||||
f"model-{i}": {"litellm_provider": "openai", "mode": "chat"} for i in range(n)
|
||||
|
|
@ -41,6 +57,7 @@ def test_count_model_entries_excludes_reserved_keys():
|
|||
m = _make_models(3)
|
||||
m["sample_spec"] = {"foo": "bar"}
|
||||
m[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []}
|
||||
m[METADATA_KEY] = dict(_STAMP)
|
||||
assert _count_model_entries(m) == 3
|
||||
|
||||
|
||||
|
|
@ -126,6 +143,39 @@ def test_finalize_with_no_block_clears_rules():
|
|||
set_fallback_generalizations(previous)
|
||||
|
||||
|
||||
def test_finalize_pops_metadata_and_records_provenance():
|
||||
finalized = _finalize_model_cost_map({**_make_models(2), METADATA_KEY: dict(_STAMP)})
|
||||
|
||||
assert METADATA_KEY not in finalized
|
||||
assert len(finalized) == 2
|
||||
provenance = get_model_cost_map_provenance()
|
||||
assert provenance["generated_at"] == _STAMP["generated_at"]
|
||||
assert provenance["source_revision"] == _STAMP["source_revision"]
|
||||
|
||||
|
||||
def test_finalize_without_metadata_clears_the_previous_stamp():
|
||||
_finalize_model_cost_map({**_make_models(2), METADATA_KEY: dict(_STAMP)})
|
||||
|
||||
_finalize_model_cost_map(_make_models(2))
|
||||
|
||||
provenance = get_model_cost_map_provenance()
|
||||
assert provenance["generated_at"] is None
|
||||
assert provenance["source_revision"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
["2026-09-07T00:00:00Z", {"generated_at": 42}, ["2026-09-07T00:00:00Z"]],
|
||||
ids=["string", "wrong_field_type", "list"],
|
||||
)
|
||||
def test_finalize_tolerates_a_malformed_metadata_block(raw):
|
||||
finalized = _finalize_model_cost_map({**_make_models(2), METADATA_KEY: raw})
|
||||
|
||||
assert METADATA_KEY not in finalized
|
||||
assert len(finalized) == 2
|
||||
assert get_model_cost_map_provenance()["generated_at"] is None
|
||||
|
||||
|
||||
def test_shipped_backup_carries_the_claude_routing_rules():
|
||||
"""The bundled backup must ship the Claude routing rules so a fresh install
|
||||
(or an offline fallback) routes unknown Claude models without code changes.
|
||||
|
|
@ -340,6 +390,10 @@ def _real_map_bytes() -> bytes:
|
|||
return json.dumps(_load_root_cost_map()).encode()
|
||||
|
||||
|
||||
def _stamped_map_bytes(stamp: dict) -> bytes:
|
||||
return json.dumps({**_load_root_cost_map(), METADATA_KEY: stamp}).encode()
|
||||
|
||||
|
||||
class _SleepRecorder:
|
||||
"""Injected in place of asyncio.sleep so tests assert waits without real delay."""
|
||||
|
||||
|
|
@ -500,6 +554,43 @@ async def test_refetch_respects_local_env_override(monkeypatch):
|
|||
assert len(result.model_cost_map) > 100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refetch_records_the_file_stamp_and_the_fetch_etag():
|
||||
"""A reload reports which revision of the map it swapped in: the ``_metadata`` stamp the file
|
||||
carries plus the ETag the fetch returned, with the stamp itself kept out of the model map."""
|
||||
client, _ = _mock_client(
|
||||
[httpx.Response(200, headers={"ETag": 'W/"abc123"'}, content=_stamped_map_bytes(_STAMP))]
|
||||
)
|
||||
|
||||
result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client)
|
||||
|
||||
assert isinstance(result, ModelCostMapReloaded)
|
||||
assert result.etag == 'W/"abc123"'
|
||||
assert METADATA_KEY not in result.model_cost_map
|
||||
assert get_model_cost_map_provenance() == {
|
||||
"generated_at": _STAMP["generated_at"],
|
||||
"source_revision": _STAMP["source_revision"],
|
||||
"etag": 'W/"abc123"',
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refetch_local_override_reports_the_bundled_stamp_without_an_etag(monkeypatch):
|
||||
"""Forcing the bundled backup after a remote reload must drop the remote ETag, since the map
|
||||
served is no longer the one that ETag identifies."""
|
||||
remote, _ = _mock_client(
|
||||
[httpx.Response(200, headers={"ETag": 'W/"remote"'}, content=_stamped_map_bytes(_STAMP))]
|
||||
)
|
||||
await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=remote)
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
|
||||
result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0))
|
||||
|
||||
assert isinstance(result, ModelCostMapReloaded)
|
||||
assert METADATA_KEY not in result.model_cost_map
|
||||
assert get_model_cost_map_provenance() == {**_load_bundled_stamp(), "etag": None}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_model_cost_map: the boot-time load retries transient failures like a reload does
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -542,7 +633,7 @@ def test_boot_load_retries_transient_failures_instead_of_falling_back():
|
|||
source = get_model_cost_map_source_info()
|
||||
assert source["source"] == "remote"
|
||||
assert source["fallback_reason"] is None
|
||||
assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY}
|
||||
assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY, METADATA_KEY}
|
||||
|
||||
|
||||
def test_boot_load_honors_retry_after_then_falls_back_after_max_attempts():
|
||||
|
|
@ -592,3 +683,39 @@ def test_boot_load_respects_local_env_override(monkeypatch):
|
|||
)
|
||||
assert len(cost_map) > 100
|
||||
assert get_model_cost_map_source_info()["is_env_forced"] is True
|
||||
|
||||
|
||||
def test_boot_load_records_the_file_stamp_and_the_fetch_etag():
|
||||
client, _ = _mock_client(
|
||||
[httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_stamped_map_bytes(_STAMP))],
|
||||
client_cls=httpx.Client,
|
||||
)
|
||||
|
||||
cost_map = get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=client)
|
||||
|
||||
assert METADATA_KEY not in cost_map
|
||||
source = get_model_cost_map_source_info()
|
||||
assert source["source"] == "remote"
|
||||
assert source["etag"] == 'W/"boot"'
|
||||
assert source["generated_at"] == _STAMP["generated_at"]
|
||||
assert source["source_revision"] == _STAMP["source_revision"]
|
||||
assert source["loaded_at"] is not None
|
||||
|
||||
|
||||
def test_boot_load_fallback_to_the_backup_drops_the_remote_etag():
|
||||
"""A boot that lands on the bundled backup reports the backup's own stamp and no ETag, even
|
||||
when an earlier load in the same process had fetched the remote map."""
|
||||
remote, _ = _mock_client(
|
||||
[httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_stamped_map_bytes(_STAMP))],
|
||||
client_cls=httpx.Client,
|
||||
)
|
||||
get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=remote)
|
||||
failing, _ = _mock_client([httpx.Response(404)], client_cls=httpx.Client)
|
||||
|
||||
cost_map = get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=failing)
|
||||
|
||||
assert METADATA_KEY not in cost_map
|
||||
source = get_model_cost_map_source_info()
|
||||
assert source["source"] == "local"
|
||||
assert source["etag"] is None
|
||||
assert {"generated_at": source["generated_at"], "source_revision": source["source_revision"]} == _load_bundled_stamp()
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ Routes covered:
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
|
||||
|
|
@ -20,6 +21,13 @@ from .conftest import VOLATILE_KEYS, normalize
|
|||
# dict-equality assertions remain stable.
|
||||
_VOLATILE = VOLATILE_KEYS | frozenset({"timestamp"})
|
||||
|
||||
_PROVENANCE = {
|
||||
"generated_at": "2026-09-07T00:00:00Z",
|
||||
"source_revision": "0123456789abcdef0123456789abcdef01234567",
|
||||
"etag": 'W/"cost-map-etag"',
|
||||
}
|
||||
_ROOT_COST_MAP = Path(__file__).resolve().parents[4] / "model_prices_and_context_window.json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
|
|
@ -42,6 +50,14 @@ def _attach_litellm_config(mock_prisma):
|
|||
return table
|
||||
|
||||
|
||||
def _pin_provenance(monkeypatch):
|
||||
"""Fix what this process reports as its cost map revision, independent of the map loaded at import."""
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map_provenance",
|
||||
lambda: dict(_PROVENANCE),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# POST /reload/model_cost_map
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -55,6 +71,7 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
|
|||
|
||||
table = _attach_litellm_config(mock_prisma)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
_pin_provenance(monkeypatch)
|
||||
|
||||
fake_cost_map = {"gpt-4": {"input_cost": 0.03}, "gpt-3.5": {"input_cost": 0.002}}
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -83,6 +100,7 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
|
|||
"status": "success",
|
||||
"models_count": 2,
|
||||
"timestamp": "<VOLATILE>",
|
||||
**_PROVENANCE,
|
||||
}
|
||||
assert table.upsert.await_count == 1
|
||||
update_payload = table.upsert.await_args.kwargs["data"]["update"]
|
||||
|
|
@ -90,6 +108,57 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
|
|||
assert update_payload["reload_revision"] == {"increment": 1}
|
||||
|
||||
|
||||
def test_reload_model_cost_map_surfaces_provenance_and_keeps_metadata_out_of_the_model_list(
|
||||
client, auth_as, monkeypatch, mock_prisma
|
||||
):
|
||||
"""A real refetch through the reload route reports the file's stamp and the fetch ETag on every
|
||||
status surface, while the ``_metadata`` block never shows up as a model anywhere."""
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
_attach_litellm_config(mock_prisma)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False)
|
||||
stamped = {**json.loads(_ROOT_COST_MAP.read_text()), "_metadata": {k: v for k, v in _PROVENANCE.items() if k != "etag"}}
|
||||
served = httpx.Response(200, headers={"ETag": _PROVENANCE["etag"]}, content=json.dumps(stamped).encode())
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.get_model_cost_map._default_reload_client",
|
||||
lambda: httpx.AsyncClient(transport=httpx.MockTransport(lambda request: served)),
|
||||
)
|
||||
monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None)
|
||||
monkeypatch.setattr("litellm.model_cost", {}, raising=False)
|
||||
|
||||
async def _fake_invalidate(name):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(ps, "invalidate_config_param", _fake_invalidate)
|
||||
|
||||
with auth_as(LitellmUserRoles.PROXY_ADMIN):
|
||||
reload_response = client.post("/reload/model_cost_map")
|
||||
source_response = client.get("/model/cost_map/source")
|
||||
status_response = client.get("/schedule/model_cost_map_reload/status")
|
||||
public_response = client.get("/public/litellm_model_cost_map")
|
||||
|
||||
assert reload_response.status_code == 200
|
||||
reload_body = reload_response.json()
|
||||
assert {key: reload_body[key] for key in _PROVENANCE} == _PROVENANCE
|
||||
assert source_response.status_code == 200
|
||||
source_body = source_response.json()
|
||||
assert {key: source_body[key] for key in _PROVENANCE} == _PROVENANCE
|
||||
assert source_body["source"] == "remote"
|
||||
assert status_response.status_code == 200
|
||||
assert {key: status_response.json()[key] for key in _PROVENANCE} == _PROVENANCE
|
||||
assert public_response.status_code == 200
|
||||
public_body = public_response.json()
|
||||
assert "_metadata" not in public_body
|
||||
assert "_metadata" not in litellm.model_cost
|
||||
assert "gpt-4o" in public_body
|
||||
assert reload_body["models_count"] == len(litellm.model_cost)
|
||||
|
||||
|
||||
def test_reload_model_cost_map_fetch_failure_502_keeps_map(
|
||||
client, auth_as, monkeypatch, mock_prisma
|
||||
):
|
||||
|
|
@ -270,11 +339,12 @@ def test_cancel_model_cost_map_reload_no_db_500(client, auth_as, monkeypatch):
|
|||
def test_get_model_cost_map_reload_status_no_db_not_scheduled(
|
||||
client, auth_as, monkeypatch
|
||||
):
|
||||
"""No prisma client → returns the not-scheduled shape (4 keys, all-null)."""
|
||||
"""No prisma client → returns the not-scheduled shape (all-null) plus the cost map provenance."""
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
monkeypatch.setattr(ps, "prisma_client", None)
|
||||
_pin_provenance(monkeypatch)
|
||||
with auth_as(LitellmUserRoles.PROXY_ADMIN):
|
||||
response = client.get("/schedule/model_cost_map_reload/status")
|
||||
assert response.status_code == 200
|
||||
|
|
@ -283,6 +353,7 @@ def test_get_model_cost_map_reload_status_no_db_not_scheduled(
|
|||
"interval_hours": None,
|
||||
"last_run": None,
|
||||
"next_run": None,
|
||||
**_PROVENANCE,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -300,6 +371,7 @@ def test_get_model_cost_map_reload_status_scheduled(
|
|||
config_row.last_run_at = None
|
||||
table.find_unique = AsyncMock(return_value=config_row)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
_pin_provenance(monkeypatch)
|
||||
|
||||
with auth_as(LitellmUserRoles.PROXY_ADMIN):
|
||||
response = client.get("/schedule/model_cost_map_reload/status")
|
||||
|
|
@ -309,6 +381,7 @@ def test_get_model_cost_map_reload_status_scheduled(
|
|||
"interval_hours": 12,
|
||||
"last_run": None,
|
||||
"next_run": None,
|
||||
**_PROVENANCE,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -328,6 +401,7 @@ def test_get_model_cost_map_reload_status_reports_persisted_last_run(
|
|||
config_row.last_run_at = datetime(2024, 1, 1, 6, 0, 0, tzinfo=timezone.utc)
|
||||
table.find_unique = AsyncMock(return_value=config_row)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
_pin_provenance(monkeypatch)
|
||||
|
||||
with auth_as(LitellmUserRoles.PROXY_ADMIN):
|
||||
response = client.get("/schedule/model_cost_map_reload/status")
|
||||
|
|
@ -337,6 +411,7 @@ def test_get_model_cost_map_reload_status_reports_persisted_last_run(
|
|||
"interval_hours": 6,
|
||||
"last_run": "2024-01-01T06:00:00+00:00",
|
||||
"next_run": "2024-01-01T12:00:00+00:00",
|
||||
**_PROVENANCE,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -356,6 +431,7 @@ def test_get_model_cost_map_reload_status_no_config_not_scheduled(
|
|||
config_row.last_run_at = None
|
||||
table.find_unique = AsyncMock(return_value=config_row)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
_pin_provenance(monkeypatch)
|
||||
|
||||
with auth_as(LitellmUserRoles.PROXY_ADMIN):
|
||||
response = client.get("/schedule/model_cost_map_reload/status")
|
||||
|
|
@ -365,6 +441,7 @@ def test_get_model_cost_map_reload_status_no_config_not_scheduled(
|
|||
"interval_hours": None,
|
||||
"last_run": None,
|
||||
"next_run": None,
|
||||
**_PROVENANCE,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -391,6 +468,8 @@ def test_get_model_cost_map_source_happy(client, auth_as, monkeypatch):
|
|||
"url": "https://example.invalid/cost_map.json",
|
||||
"is_env_forced": False,
|
||||
"fallback_reason": None,
|
||||
"loaded_at": "2026-09-07T01:02:03+00:00",
|
||||
**_PROVENANCE,
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map_source_info",
|
||||
|
|
@ -406,6 +485,8 @@ def test_get_model_cost_map_source_happy(client, auth_as, monkeypatch):
|
|||
"url": "https://example.invalid/cost_map.json",
|
||||
"is_env_forced": False,
|
||||
"fallback_reason": None,
|
||||
"loaded_at": "2026-09-07T01:02:03+00:00",
|
||||
**_PROVENANCE,
|
||||
"model_count": 3,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
"""Tests for .github/scripts/auto_update_price_and_context_window_file.py."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
_REPO_ROOT: Final = Path(__file__).resolve().parents[2]
|
||||
_MODULE_PATH: Final = _REPO_ROOT / ".github" / "scripts" / "auto_update_price_and_context_window_file.py"
|
||||
_spec: Final = importlib.util.spec_from_file_location("auto_update_price_and_context_window_file", _MODULE_PATH)
|
||||
script: Final = importlib.util.module_from_spec(_spec)
|
||||
sys.modules[_spec.name] = script
|
||||
_spec.loader.exec_module(script)
|
||||
|
||||
_LOCAL_FILE: Final = "model_prices_and_context_window.json"
|
||||
_GENERATED_AT: Final = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z")
|
||||
|
||||
|
||||
def _openrouter_row(model_id: str) -> dict:
|
||||
return {"id": model_id, "context_length": 8192, "pricing": {"prompt": "0.000001", "completion": "0.000002"}}
|
||||
|
||||
|
||||
def _serve(openrouter_rows: list) -> object:
|
||||
async def fetch_data(url: str) -> list:
|
||||
return openrouter_rows if "openrouter" in url else []
|
||||
|
||||
return fetch_data
|
||||
|
||||
|
||||
def _read_local(tmp_path: Path) -> dict:
|
||||
return json.loads((tmp_path / _LOCAL_FILE).read_text())
|
||||
|
||||
|
||||
def test_main_stamps_provenance_only_when_the_sync_changed_the_file(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("GITHUB_SHA", "feedface")
|
||||
monkeypatch.setattr(script, "fetch_data", _serve([_openrouter_row("acme/x")]))
|
||||
(tmp_path / _LOCAL_FILE).write_text(json.dumps({"sample_spec": {"input_cost_per_token": "USD"}}, indent=4) + "\n")
|
||||
|
||||
script.main()
|
||||
|
||||
written = _read_local(tmp_path)
|
||||
assert written["openrouter/acme/x"]["litellm_provider"] == "openrouter"
|
||||
assert written["_metadata"]["source_revision"] == "feedface"
|
||||
assert _GENERATED_AT.fullmatch(written["_metadata"]["generated_at"])
|
||||
|
||||
sentinel = {**written, "_metadata": {**written["_metadata"], "generated_at": "2000-01-01T00:00:00Z"}}
|
||||
(tmp_path / _LOCAL_FILE).write_text(json.dumps(sentinel, indent=4) + "\n")
|
||||
|
||||
script.main()
|
||||
|
||||
assert _read_local(tmp_path) == sentinel
|
||||
|
|
@ -141,6 +141,26 @@ def test_bot_may_not_change_special_root_keys() -> None:
|
|||
assert _failures(head) == ("bot PRs may not change fallback_generalizations",)
|
||||
|
||||
|
||||
STAMP: Final = {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef0123456789abcdef01234567"}
|
||||
|
||||
|
||||
def test_bot_may_stamp_and_restamp_metadata() -> None:
|
||||
stamped = _snapshot({**BASE_MAP, "_metadata": STAMP})
|
||||
assert _failures(stamped) == ()
|
||||
assert _failures(stamped, bot=False) == ()
|
||||
|
||||
restamped = _snapshot(
|
||||
{
|
||||
**BASE_MAP,
|
||||
"_metadata": {**STAMP, "generated_at": "2026-09-14T00:00:00Z"},
|
||||
"fallback_generalizations": {"rules": []},
|
||||
}
|
||||
)
|
||||
assert guard.guard_failures(stamped, restamped, MAP_FILES, True) == (
|
||||
"bot PRs may not change fallback_generalizations",
|
||||
)
|
||||
|
||||
|
||||
def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str:
|
||||
text = _serialize(cost_map)
|
||||
(repo / guard.COST_MAP_PATH).write_text(text)
|
||||
|
|
|
|||
|
|
@ -98,6 +98,25 @@ def test_schema_rejects_malformed_entries(committed_schema: dict, entry: dict):
|
|||
assert not validator.is_valid({"some-model": entry})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"metadata",
|
||||
[
|
||||
"2026-09-07T00:00:00Z",
|
||||
{"generated_at": "2026-09-07T00:00:00Z"},
|
||||
{"source_revision": "0123456789abcdef"},
|
||||
{"generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef", "author": "bot"},
|
||||
],
|
||||
ids=["not_an_object", "missing_revision", "missing_generated_at", "unknown_field"],
|
||||
)
|
||||
def test_schema_rejects_a_malformed_metadata_block(committed_schema: dict, metadata: object):
|
||||
assert not build_validator(committed_schema).is_valid({"_metadata": metadata})
|
||||
|
||||
|
||||
def test_schema_accepts_the_provenance_stamp_as_a_non_model_root_key(committed_schema: dict):
|
||||
stamp = {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "0123456789abcdef0123456789abcdef01234567"}
|
||||
assert build_validator(committed_schema).is_valid({"_metadata": stamp})
|
||||
|
||||
|
||||
def test_schema_accepts_minimal_and_unknown_optional_fields(committed_schema: dict):
|
||||
validator = build_validator(committed_schema)
|
||||
assert validator.is_valid({"some-model": {"litellm_provider": "openai"}})
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
|
||||
|
|
@ -369,6 +370,58 @@ def test_sync_is_idempotent_over_the_repo_cost_map() -> None:
|
|||
assert second.cost_map == first.cost_map
|
||||
|
||||
|
||||
def test_stamp_metadata_adds_the_provenance_block_without_touching_models() -> None:
|
||||
cost_map = {"sample_spec": {"input_cost_per_token": "USD"}, "together_ai/acme/x": {"mode": "chat"}}
|
||||
|
||||
stamped = sync.stamp_metadata(cost_map, "2026-09-07T00:00:00Z", "feedface")
|
||||
|
||||
assert stamped["_metadata"] == {"generated_at": "2026-09-07T00:00:00Z", "source_revision": "feedface"}
|
||||
assert {key: value for key, value in stamped.items() if key != "_metadata"} == cost_map
|
||||
assert "_metadata" not in cost_map
|
||||
|
||||
|
||||
def _write_registry(repo_root: Path, cost_map: dict) -> None:
|
||||
for relpath in sync.COST_MAP_RELPATHS:
|
||||
target = repo_root / relpath
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(json.dumps(cost_map, indent=4) + "\n")
|
||||
|
||||
|
||||
def _read_registries(repo_root: Path) -> tuple[dict, ...]:
|
||||
return tuple(json.loads((repo_root / relpath).read_text()) for relpath in sync.COST_MAP_RELPATHS)
|
||||
|
||||
|
||||
def test_write_stamps_provenance_into_both_files_only_when_the_sync_changed_them(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setenv("GITHUB_SHA", "feedface")
|
||||
cost_map = json.loads((ROOT / "model_prices_and_context_window.json").read_text())
|
||||
dropped = next(f"together_ai/{model.id}" for model in RECORDED_CATALOG if f"together_ai/{model.id}" in cost_map)
|
||||
_write_registry(tmp_path, {key: value for key, value in cost_map.items() if key not in {dropped, "_metadata"}})
|
||||
argv = (
|
||||
"--write",
|
||||
"--models-json",
|
||||
str(FIXTURES / "models_serverless.json"),
|
||||
"--deprecations-md",
|
||||
str(FIXTURES / "deprecations.md"),
|
||||
"--repo-root",
|
||||
str(tmp_path),
|
||||
)
|
||||
|
||||
assert sync.main(argv) == 0
|
||||
|
||||
written = _read_registries(tmp_path)
|
||||
assert written[0] == written[1]
|
||||
assert dropped in written[0]
|
||||
assert written[0]["_metadata"]["source_revision"] == "feedface"
|
||||
assert re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", written[0]["_metadata"]["generated_at"])
|
||||
|
||||
sentinel = {**written[0], "_metadata": {**written[0]["_metadata"], "generated_at": "2000-01-01T00:00:00Z"}}
|
||||
_write_registry(tmp_path, sentinel)
|
||||
|
||||
assert sync.main(argv) == 0
|
||||
|
||||
assert _read_registries(tmp_path) == (sentinel, sentinel)
|
||||
|
||||
|
||||
def test_pr_body_lists_every_section_and_the_skipped_types() -> None:
|
||||
outcome = sync.compute_sync({}, RECORDED_CATALOG, RECORDED_DOC)
|
||||
body = sync.render_pr_body(outcome)
|
||||
|
|
|
|||
|
|
@ -32,8 +32,18 @@ const remoteSource = {
|
|||
url: "https://pricing.example.test/model_prices.json",
|
||||
is_env_forced: false,
|
||||
fallback_reason: null,
|
||||
loaded_at: null,
|
||||
generated_at: null,
|
||||
source_revision: null,
|
||||
etag: null,
|
||||
model_count: 1234,
|
||||
};
|
||||
const provenance = {
|
||||
loaded_at: "2026-09-07T10:00:00Z",
|
||||
generated_at: "2026-09-06T23:38:47Z",
|
||||
source_revision: "cd681a573fd9f5b6f15a1355f46178e4e9d374d2",
|
||||
etag: 'W/"eb8e9a53f4cc284b"',
|
||||
};
|
||||
|
||||
describe("PriceDataReload", () => {
|
||||
beforeEach(() => {
|
||||
|
|
@ -51,6 +61,30 @@ describe("PriceDataReload", () => {
|
|||
expect(screen.getByText("No periodic reload scheduled")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows which revision of the cost map is loaded when the source reports one", async () => {
|
||||
vi.mocked(getModelCostMapSource).mockResolvedValue({ ...remoteSource, ...provenance } as never);
|
||||
render(<PriceDataReload accessToken="sk-test" />);
|
||||
|
||||
expect(await screen.findByText("Source revision:")).toBeInTheDocument();
|
||||
expect(screen.getByText("cd681a573fd9")).toBeInTheDocument();
|
||||
expect(screen.getByText("ETag:")).toBeInTheDocument();
|
||||
expect(screen.getByText('W/"eb8e9a53f4cc284b"')).toBeInTheDocument();
|
||||
expect(screen.getByText("Generated at:")).toBeInTheDocument();
|
||||
expect(screen.getByText(new Date(provenance.generated_at).toLocaleString())).toBeInTheDocument();
|
||||
expect(screen.getByText("Loaded at:")).toBeInTheDocument();
|
||||
expect(screen.getByText(new Date(provenance.loaded_at).toLocaleString())).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("hides the provenance rows when the loaded map carries no stamp", async () => {
|
||||
render(<PriceDataReload accessToken="sk-test" />);
|
||||
|
||||
expect(await screen.findByText("Pricing Data Source")).toBeInTheDocument();
|
||||
expect(screen.queryByText("Generated at:")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Source revision:")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("ETag:")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Loaded at:")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("confirms an immediate reload and refreshes dependent data", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onReloadSuccess = vi.fn();
|
||||
|
|
|
|||
|
|
@ -49,9 +49,17 @@ interface CostMapSourceInfo {
|
|||
url: string | null;
|
||||
is_env_forced: boolean;
|
||||
fallback_reason: string | null;
|
||||
loaded_at: string | null;
|
||||
generated_at: string | null;
|
||||
source_revision: string | null;
|
||||
etag: string | null;
|
||||
model_count: number;
|
||||
}
|
||||
|
||||
const SHORT_REVISION_LENGTH = 12;
|
||||
|
||||
const shortRevision = (revision: string) => revision.slice(0, SHORT_REVISION_LENGTH);
|
||||
|
||||
const EMPTY_RELOAD_STATUS: ReloadStatus = {
|
||||
scheduled: false,
|
||||
interval_hours: null,
|
||||
|
|
@ -89,6 +97,55 @@ const isValidReloadInterval = (value: number) => {
|
|||
return value >= 1 && value <= 168;
|
||||
};
|
||||
|
||||
const formatDateTime = (dateTimeString: string | null) => {
|
||||
if (!dateTimeString) return "Never";
|
||||
try {
|
||||
return new Date(dateTimeString).toLocaleString();
|
||||
} catch {
|
||||
return dateTimeString;
|
||||
}
|
||||
};
|
||||
|
||||
const CostMapProvenanceRows: React.FC<{ sourceInfo: CostMapSourceInfo }> = ({ sourceInfo }) => (
|
||||
<>
|
||||
{sourceInfo.generated_at && (
|
||||
<div className="flex items-center justify-between text-xs">
|
||||
<span className="text-muted-foreground">Generated at:</span>
|
||||
<span className="font-medium">{formatDateTime(sourceInfo.generated_at)}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{sourceInfo.source_revision && (
|
||||
<div className="flex items-center justify-between gap-2 text-xs">
|
||||
<span className="text-muted-foreground">Source revision:</span>
|
||||
<Tooltip>
|
||||
<TooltipTrigger render={<code className="font-mono" />}>
|
||||
{shortRevision(sourceInfo.source_revision)}
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{sourceInfo.source_revision}</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{sourceInfo.etag && (
|
||||
<div className="flex items-center justify-between gap-2 text-xs">
|
||||
<span className="text-muted-foreground">ETag:</span>
|
||||
<Tooltip>
|
||||
<TooltipTrigger render={<code className="max-w-60 truncate font-mono" />}>{sourceInfo.etag}</TooltipTrigger>
|
||||
<TooltipContent>{sourceInfo.etag}</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{sourceInfo.loaded_at && (
|
||||
<div className="flex items-center justify-between text-xs">
|
||||
<span className="text-muted-foreground">Loaded at:</span>
|
||||
<span className="font-medium">{formatDateTime(sourceInfo.loaded_at)}</span>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
|
||||
const PriceDataReload: React.FC<PriceDataReloadProps> = ({
|
||||
accessToken,
|
||||
onReloadSuccess,
|
||||
|
|
@ -227,15 +284,6 @@ const PriceDataReload: React.FC<PriceDataReloadProps> = ({
|
|||
}
|
||||
};
|
||||
|
||||
const formatDateTime = (dateTimeString: string | null) => {
|
||||
if (!dateTimeString) return "Never";
|
||||
try {
|
||||
return new Date(dateTimeString).toLocaleString();
|
||||
} catch {
|
||||
return dateTimeString;
|
||||
}
|
||||
};
|
||||
|
||||
const getStatusText = () => {
|
||||
if (!reloadStatus?.scheduled) return "Not scheduled";
|
||||
if (!reloadStatus.last_run) return "Ready";
|
||||
|
|
@ -334,6 +382,8 @@ const PriceDataReload: React.FC<PriceDataReloadProps> = ({
|
|||
</div>
|
||||
)}
|
||||
|
||||
<CostMapProvenanceRows sourceInfo={sourceInfo} />
|
||||
|
||||
{sourceInfo.is_env_forced && (
|
||||
<div className="flex items-center gap-1.5 text-xs text-muted-foreground">
|
||||
<Info className="size-3.5 shrink-0" />
|
||||
|
|
|
|||
3
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
3
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -8684,6 +8684,9 @@ export interface paths {
|
|||
* - url: the remote URL that was attempted (null when env-forced local)
|
||||
* - is_env_forced: true if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage
|
||||
* - fallback_reason: human-readable reason why remote failed (null on success)
|
||||
* - loaded_at: when this pod last loaded the map
|
||||
* - generated_at, source_revision: the _metadata stamp inside the loaded file
|
||||
* - etag: the ETag of the remote fetch (null for the bundled backup)
|
||||
* - model_count: number of models in the currently loaded cost map
|
||||
*/
|
||||
get: operations["get_model_cost_map_source_model_cost_map_source_get"];
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue