mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
Merge branch 'litellm_internal_staging' into litellm_/release-version-bump-787548
This commit is contained in:
yuneng-jiang
GitHub
commit
86ee031217
21 changed files with 706 additions and 96 deletions
|
|
@ -9,17 +9,19 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime, timezone
|
||||
from importlib.resources import files
|
||||
from typing import Final, Protocol
|
||||
|
||||
import httpx
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm.constants import (
|
||||
|
|
@ -42,6 +44,10 @@ def _count_model_entries(model_cost: dict) -> int:
|
|||
return sum(1 for key in model_cost if key not in RESERVED_TOP_LEVEL_KEYS)
|
||||
|
||||
|
||||
def git_blob_id(body: bytes) -> str:
|
||||
return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest()
|
||||
|
||||
|
||||
class GetModelCostMap:
|
||||
"""
|
||||
Handles fetching, validating, and loading the model cost map.
|
||||
|
|
@ -53,15 +59,24 @@ class GetModelCostMap:
|
|||
|
||||
_backup_model_count: int = -1 # -1 = not yet loaded
|
||||
|
||||
@staticmethod
|
||||
def read_local_model_cost_map_bytes() -> bytes:
|
||||
return files("litellm").joinpath("model_prices_and_context_window_backup.json").read_bytes()
|
||||
|
||||
@staticmethod
|
||||
def read_local_model_cost_map_text() -> str:
|
||||
return files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text(encoding="utf-8")
|
||||
return GetModelCostMap.read_local_model_cost_map_bytes().decode("utf-8")
|
||||
|
||||
@staticmethod
|
||||
def load_local_model_cost_map_with_revision() -> "ModelCostMapReloaded":
|
||||
body: Final = GetModelCostMap.read_local_model_cost_map_bytes()
|
||||
content: Final = json.loads(body)
|
||||
return ModelCostMapReloaded(model_cost_map=content, revision=git_blob_id(body))
|
||||
|
||||
@staticmethod
|
||||
def load_local_model_cost_map() -> dict:
|
||||
"""Load the local backup model cost map bundled with the package."""
|
||||
content: Final = json.loads(GetModelCostMap.read_local_model_cost_map_text())
|
||||
return content
|
||||
return GetModelCostMap.load_local_model_cost_map_with_revision().model_cost_map
|
||||
|
||||
@classmethod
|
||||
def _get_backup_model_count(cls) -> int:
|
||||
|
|
@ -166,6 +181,8 @@ MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class ModelCostMapReloaded:
|
||||
model_cost_map: dict # mutable-ok: adopted as litellm.model_cost, whose consumer contract is a plain mutable dict
|
||||
revision: str | None = None
|
||||
etag: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -254,7 +271,9 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp
|
|||
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
|
||||
if not isinstance(parsed, dict):
|
||||
return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
|
||||
return ModelCostMapReloaded(model_cost_map=parsed)
|
||||
return ModelCostMapReloaded(
|
||||
model_cost_map=parsed, revision=git_blob_id(response.content), etag=response.headers.get("etag")
|
||||
)
|
||||
|
||||
|
||||
def _next_retry_wait(
|
||||
|
|
@ -328,13 +347,12 @@ async def refetch_model_cost_map(
|
|||
map they already have.
|
||||
"""
|
||||
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.url = None
|
||||
_cost_map_source_info.is_env_forced = True
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return ModelCostMapReloaded(
|
||||
model_cost_map=_finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map())
|
||||
)
|
||||
return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision())
|
||||
|
||||
result: Final = await _fetch_remote_model_cost_map_with_retry(
|
||||
url=url,
|
||||
|
|
@ -355,11 +373,12 @@ async def refetch_model_cost_map(
|
|||
backup_model_count=GetModelCostMap._get_backup_model_count(),
|
||||
):
|
||||
return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} failed integrity validation")
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
_cost_map_source_info.source = "remote"
|
||||
_cost_map_source_info.url = url
|
||||
_cost_map_source_info.is_env_forced = False
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return ModelCostMapReloaded(model_cost_map=_finalize_model_cost_map(result.model_cost_map))
|
||||
return _finalize_loaded_model_cost_map(result)
|
||||
|
||||
|
||||
class ModelCostMapSourceInfo:
|
||||
|
|
@ -370,13 +389,35 @@ class ModelCostMapSourceInfo:
|
|||
is_env_forced: bool = False
|
||||
fallback_reason: str | None = None
|
||||
loaded_at: "datetime | None" = None
|
||||
source_revision: str | None = None
|
||||
etag: str | None = None
|
||||
|
||||
|
||||
# Module-level singleton tracking the source of the current cost map
|
||||
_cost_map_source_info: Final = ModelCostMapSourceInfo()
|
||||
|
||||
|
||||
def get_model_cost_map_source_info() -> dict:
|
||||
class CostMapProvenance(TypedDict):
|
||||
source_revision: ReadOnly[str | None]
|
||||
etag: ReadOnly[str | None]
|
||||
|
||||
|
||||
class CostMapSourceInfo(CostMapProvenance):
|
||||
source: ReadOnly[str]
|
||||
url: ReadOnly[str | None]
|
||||
is_env_forced: ReadOnly[bool]
|
||||
fallback_reason: ReadOnly[str | None]
|
||||
loaded_at: ReadOnly[str | None]
|
||||
|
||||
|
||||
def get_model_cost_map_provenance() -> CostMapProvenance:
|
||||
return {
|
||||
"source_revision": _cost_map_source_info.source_revision,
|
||||
"etag": _cost_map_source_info.etag,
|
||||
}
|
||||
|
||||
|
||||
def get_model_cost_map_source_info() -> CostMapSourceInfo:
|
||||
"""
|
||||
Return metadata about where the current model cost map was loaded from.
|
||||
|
||||
|
|
@ -385,12 +426,19 @@ def get_model_cost_map_source_info() -> dict:
|
|||
- url: the remote URL attempted (or None for local-only)
|
||||
- is_env_forced: True if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage
|
||||
- fallback_reason: human-readable reason if remote failed and local was used
|
||||
- loaded_at: ISO 8601 time this process last loaded the map
|
||||
- source_revision: git blob id of the loaded file's bytes
|
||||
- etag: the ETag of the remote fetch (None for the bundled backup)
|
||||
"""
|
||||
loaded_at: Final = _cost_map_source_info.loaded_at
|
||||
return {
|
||||
"source": _cost_map_source_info.source,
|
||||
"url": _cost_map_source_info.url,
|
||||
"is_env_forced": _cost_map_source_info.is_env_forced,
|
||||
"fallback_reason": _cost_map_source_info.fallback_reason,
|
||||
"loaded_at": loaded_at.isoformat() if loaded_at is not None else None,
|
||||
"source_revision": _cost_map_source_info.source_revision,
|
||||
"etag": _cost_map_source_info.etag,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -466,6 +514,12 @@ def _finalize_model_cost_map(model_cost: dict) -> dict:
|
|||
return _expand_model_aliases(model_cost)
|
||||
|
||||
|
||||
def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded:
|
||||
_cost_map_source_info.source_revision = loaded.revision
|
||||
_cost_map_source_info.etag = loaded.etag
|
||||
return replace(loaded, model_cost_map=_finalize_model_cost_map(loaded.model_cost_map))
|
||||
|
||||
|
||||
def get_model_cost_map(
|
||||
url: str,
|
||||
timeout: int = 5,
|
||||
|
|
@ -494,7 +548,7 @@ def get_model_cost_map(
|
|||
_cost_map_source_info.url = None
|
||||
_cost_map_source_info.is_env_forced = True
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map())
|
||||
return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map
|
||||
|
||||
_cost_map_source_info.url = url
|
||||
_cost_map_source_info.is_env_forced = False
|
||||
|
|
@ -515,7 +569,7 @@ def get_model_cost_map(
|
|||
)
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.fallback_reason = f"Remote fetch failed: {result.reason}"
|
||||
return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map())
|
||||
return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map
|
||||
content: Final = result.model_cost_map
|
||||
|
||||
# Validate using cached count (cheap int comparison, no file I/O)
|
||||
|
|
@ -529,8 +583,8 @@ def get_model_cost_map(
|
|||
)
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.fallback_reason = "Remote data failed integrity validation"
|
||||
return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map())
|
||||
return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map
|
||||
|
||||
_cost_map_source_info.source = "remote"
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return _finalize_model_cost_map(content)
|
||||
return _finalize_loaded_model_cost_map(result).model_cost_map
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ from collections.abc import Mapping
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.types.utils import ModelInfo
|
||||
|
||||
|
|
@ -21,10 +23,27 @@ _EFFORT_DEGRADATION_CHAIN: Final[Mapping[str, tuple[str, ...]]] = MappingProxyTy
|
|||
_THINKING_OFF: Final = "none"
|
||||
|
||||
|
||||
class _ClaudeCodeUserId(BaseModel):
|
||||
"""The JSON Claude Code packs into ``metadata.user_id``; only ``session_id`` is per conversation."""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
session_id: str
|
||||
|
||||
|
||||
def prompt_cache_key_from_user_id(user_id: object) -> str | None:
|
||||
if user_id is None:
|
||||
"""The per-session key Claude Code carries inside ``metadata.user_id``, or nothing.
|
||||
|
||||
Anthropic defines ``user_id`` as an opaque end-user id, so a plain string names a person, not
|
||||
a conversation. Keying the provider cache on it pins every parallel session and subagent of that
|
||||
person to one slot, which caches worse than the provider's own prompt-prefix hashing does.
|
||||
"""
|
||||
if not isinstance(user_id, str):
|
||||
return None
|
||||
try:
|
||||
return _ClaudeCodeUserId.model_validate_json(user_id).session_id[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None
|
||||
except ValidationError:
|
||||
return None
|
||||
return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None
|
||||
|
||||
|
||||
def litellm_logging_obj_from_kwargs(kwargs: Mapping[str, object]) -> "LiteLLMLoggingObject | None":
|
||||
|
|
|
|||
|
|
@ -2836,7 +2836,6 @@ async def _authorize_authenticated_request(
|
|||
return None
|
||||
|
||||
|
||||
@tracer.wrap()
|
||||
def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth, request: Request | None = None) -> None:
|
||||
"""Anchor the OTLP destinations this key or team overrides its traces to.
|
||||
|
||||
|
|
@ -2874,6 +2873,7 @@ def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth, request: Reque
|
|||
verbose_proxy_logger.debug("OTel V2: tenant destination resolution failed: %s", exc)
|
||||
|
||||
|
||||
@tracer.wrap()
|
||||
async def user_api_key_auth(
|
||||
request: Request,
|
||||
api_key: str = fastapi.Security(api_key_header),
|
||||
|
|
|
|||
|
|
@ -1041,7 +1041,9 @@ def resolve_tenant_otel_destinations(
|
|||
the request has an outcome, so honouring the filter would mean holding every span
|
||||
back until the call finishes. Those entries keep today's behaviour instead, where
|
||||
the tenant's credentials reach the backend through per-request tracer routing and
|
||||
the operator's exporter is left alone.
|
||||
the operator's exporter is left alone. Its ``callback_vars`` still take part in the
|
||||
merge for a backend another entry made eligible, so the destination carries the
|
||||
same credentials the runtime parser resolves for that request.
|
||||
|
||||
A backend the request disabled dynamically, through the key's
|
||||
``litellm_disabled_callbacks`` or the ``x-litellm-disable-callbacks`` header in
|
||||
|
|
@ -1069,12 +1071,13 @@ def resolve_tenant_otel_destinations(
|
|||
callback
|
||||
for item in entries
|
||||
if (callback := _get_validated_callback_metadata(item=item, source="otel-destination")) is not None
|
||||
if callback.callback_type != "failure"
|
||||
if callback.callback_name.lower() not in disabled
|
||||
)
|
||||
return tuple(
|
||||
destination
|
||||
for name in dict.fromkeys(callback.callback_name for callback in callbacks)
|
||||
for name in dict.fromkeys(
|
||||
callback.callback_name for callback in callbacks if callback.callback_type != "failure"
|
||||
)
|
||||
if (
|
||||
destination := destination_for(
|
||||
name,
|
||||
|
|
|
|||
|
|
@ -17750,6 +17750,7 @@ async def reload_model_cost_map(
|
|||
# Immediately reload the model cost map in the current pod
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
ModelCostMapReloadUnavailable,
|
||||
get_model_cost_map_provenance,
|
||||
refetch_model_cost_map,
|
||||
)
|
||||
|
||||
|
|
@ -17763,6 +17764,7 @@ async def reload_model_cost_map(
|
|||
models_count = _swap_in_model_cost_map(reload_result.model_cost_map)
|
||||
current_time = utc_now()
|
||||
proxy_config.model_cost_map_loaded_at = current_time
|
||||
provenance: Final = get_model_cost_map_provenance()
|
||||
|
||||
# Publish a new revision so every other pod reloads on its next poll; this pod has
|
||||
# already served it, so adopt it here rather than reloading again a tick later
|
||||
|
|
@ -17777,6 +17779,7 @@ async def reload_model_cost_map(
|
|||
"status": "success",
|
||||
"models_count": models_count,
|
||||
"timestamp": current_time.isoformat(),
|
||||
**provenance,
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
|
|
@ -17897,12 +17900,17 @@ async def get_model_cost_map_reload_status(
|
|||
|
||||
try:
|
||||
global prisma_client
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
get_model_cost_map_provenance,
|
||||
)
|
||||
|
||||
provenance: Final = get_model_cost_map_provenance()
|
||||
if prisma_client is None:
|
||||
verbose_proxy_logger.info("No database connection, returning not scheduled")
|
||||
return reload_schedule_status(None)
|
||||
return {**reload_schedule_status(None), **provenance}
|
||||
|
||||
return reload_schedule_status(await read_reload_schedule(prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME))
|
||||
schedule: Final = await read_reload_schedule(prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME)
|
||||
return {**reload_schedule_status(schedule), **provenance}
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Failed to get model cost map reload status: %s", e)
|
||||
raise HTTPException(
|
||||
|
|
@ -17930,6 +17938,9 @@ async def get_model_cost_map_source(
|
|||
- url: the remote URL that was attempted (null when env-forced local)
|
||||
- is_env_forced: true if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage
|
||||
- fallback_reason: human-readable reason why remote failed (null on success)
|
||||
- loaded_at: when this pod last loaded the map
|
||||
- source_revision: git blob id of the loaded file, what git rev-parse <commit>:<path> prints for it
|
||||
- etag: the ETag of the remote fetch (null for the bundled backup)
|
||||
- model_count: number of models in the currently loaded cost map
|
||||
"""
|
||||
# Read-only source info — admin viewers can read.
|
||||
|
|
|
|||
|
|
@ -10,7 +10,12 @@
|
|||
"REASONING": ["claude-opus-5"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
"REASONING": [{ "model_name": "claude-opus-5", "litellm_params": { "reasoning_effort": "high" } }]
|
||||
"REASONING": [
|
||||
{
|
||||
"model_name": "claude-opus-5",
|
||||
"litellm_params": { "reasoning_effort": "high" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"classifier_type": "heuristic_v2",
|
||||
"escalation_keywords": ["LITELLM ESCALATE"],
|
||||
|
|
@ -23,16 +28,21 @@
|
|||
},
|
||||
"anthropic_family": {
|
||||
"label": "Anthropic Family",
|
||||
"description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex, Opus at high thinking for reasoning.",
|
||||
"description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex, Fable 5.1 at high thinking for reasoning.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["claude-haiku-4-5"],
|
||||
"MEDIUM": ["claude-sonnet-5"],
|
||||
"COMPLEX": ["claude-opus-5"],
|
||||
"REASONING": ["claude-opus-5"]
|
||||
"REASONING": ["claude-fable-5-1"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
"REASONING": [{ "model_name": "claude-opus-5", "litellm_params": { "reasoning_effort": "high" } }]
|
||||
"REASONING": [
|
||||
{
|
||||
"model_name": "claude-fable-5-1",
|
||||
"litellm_params": { "reasoning_effort": "high" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"classifier_type": "heuristic",
|
||||
"escalation_keywords": ["LITELLM ESCALATE"],
|
||||
|
|
@ -73,8 +83,18 @@
|
|||
"REASONING": ["claude-opus-5"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
"MEDIUM": [{ "model_name": "muse-spark-1.2", "litellm_params": { "reasoning_effort": "xhigh" } }],
|
||||
"COMPLEX": [{ "model_name": "kimi-k3", "litellm_params": { "reasoning_effort": "max" } }]
|
||||
"MEDIUM": [
|
||||
{
|
||||
"model_name": "muse-spark-1.2",
|
||||
"litellm_params": { "reasoning_effort": "xhigh" }
|
||||
}
|
||||
],
|
||||
"COMPLEX": [
|
||||
{
|
||||
"model_name": "kimi-k3",
|
||||
"litellm_params": { "reasoning_effort": "max" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"classifier_type": "llm",
|
||||
"classifier_llm_config": {
|
||||
|
|
@ -93,16 +113,21 @@
|
|||
},
|
||||
"openai_family": {
|
||||
"label": "OpenAI Family",
|
||||
"description": "Routes across the GPT model family: Luna for simple queries, Terra for medium, Sol for complex, Sol at xhigh thinking for reasoning.",
|
||||
"description": "Routes across the GPT model family: Luna for simple queries, Terra for medium, Sol for complex, Astra at xhigh thinking for reasoning.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["gpt-5.6-luna"],
|
||||
"MEDIUM": ["gpt-5.6-terra"],
|
||||
"COMPLEX": ["gpt-5.6-sol"],
|
||||
"REASONING": ["gpt-5.6-sol"]
|
||||
"REASONING": ["gpt-6-astra"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
"REASONING": [{ "model_name": "gpt-5.6-sol", "litellm_params": { "reasoning_effort": "xhigh" } }]
|
||||
"REASONING": [
|
||||
{
|
||||
"model_name": "gpt-6-astra",
|
||||
"litellm_params": { "reasoning_effort": "xhigh" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"classifier_type": "heuristic",
|
||||
"escalation_keywords": ["LITELLM ESCALATE"],
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[[IgnoredVulns]]
|
||||
id = "GHSA-w8v5-vhqr-4h9v"
|
||||
ignoreUntil = 2026-09-09
|
||||
ignoreUntil = 2026-10-01
|
||||
reason = "diskcache has no fixed release published; remove this entry once one exists"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@
|
|||
|
||||
import contextvars
|
||||
import time
|
||||
from base64 import b64encode
|
||||
from collections.abc import Mapping
|
||||
from functools import reduce
|
||||
from types import MappingProxyType
|
||||
|
||||
import pytest
|
||||
|
|
@ -49,8 +51,11 @@ from litellm.integrations.otel.presets.destinations import (
|
|||
destination_for,
|
||||
)
|
||||
from litellm.integrations.otel.presets.langfuse import langfuse_preset
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.litellm_pre_call_utils import resolve_tenant_otel_destinations
|
||||
from litellm.proxy._types import AddTeamCallback, UserAPIKeyAuth
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
convert_key_logging_metadata_to_callback,
|
||||
resolve_tenant_otel_destinations,
|
||||
)
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
LANGFUSE_DEST = OtelDestination(
|
||||
|
|
@ -1822,6 +1827,26 @@ class TestTenantConfigAgreement:
|
|||
|
||||
assert [d.endpoint for d in destinations] == ["http://key.local/api/public/otel"]
|
||||
|
||||
def test_a_failure_entry_still_wins_the_merge_next_to_a_success_entry(self):
|
||||
entries = [
|
||||
{**self._entry("http://team.local"), "callback_type": "success"},
|
||||
{
|
||||
**self._entry("http://key.local", langfuse_public_key="pk-failure", langfuse_secret_key="sk-failure"),
|
||||
"callback_type": "failure",
|
||||
},
|
||||
]
|
||||
runtime = reduce(
|
||||
lambda merged, entry: convert_key_logging_metadata_to_callback(AddTeamCallback(**entry), merged),
|
||||
entries,
|
||||
None,
|
||||
)
|
||||
|
||||
destinations = resolve_tenant_otel_destinations(UserAPIKeyAuth(team_metadata={"logging": entries}))
|
||||
|
||||
assert runtime.callback_vars["langfuse_host"] == "http://key.local"
|
||||
assert [d.endpoint for d in destinations] == ["http://key.local/api/public/otel"]
|
||||
assert destinations[0].headers["Authorization"] == f"Basic {b64encode(b'pk-failure:sk-failure').decode()}"
|
||||
|
||||
@pytest.fixture
|
||||
def premium(self, monkeypatch):
|
||||
from litellm.proxy import proxy_server
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ from litellm.litellm_core_utils.get_model_cost_map import (
|
|||
GetModelCostMap,
|
||||
_count_model_entries,
|
||||
_finalize_model_cost_map,
|
||||
get_model_cost_map_provenance,
|
||||
git_blob_id,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -31,6 +33,16 @@ def _load_root_cost_map() -> dict:
|
|||
return json.load(f)
|
||||
|
||||
|
||||
def _bundled_blob_id() -> str:
|
||||
path = os.path.join(os.path.dirname(__file__), "../../../litellm/model_prices_and_context_window_backup.json")
|
||||
with open(path, "rb") as f:
|
||||
return git_blob_id(f.read())
|
||||
|
||||
|
||||
def test_git_blob_id_is_what_git_hash_object_prints():
|
||||
assert git_blob_id(b'{"gpt-5.4-mini": {"mode": "chat"}}\n') == "18b9a8381e13a3b38a2128f184f631f95829e987"
|
||||
|
||||
|
||||
def _make_models(n: int) -> dict:
|
||||
return {
|
||||
f"model-{i}": {"litellm_provider": "openai", "mode": "chat"} for i in range(n)
|
||||
|
|
@ -298,14 +310,13 @@ def test_openrouter_catalog_costs_match_live_headline_rates(cost_map: dict):
|
|||
assert entry["output_cost_per_token"] != stale_out, model
|
||||
|
||||
|
||||
def test_get_model_cost_map_stamps_loaded_at(monkeypatch):
|
||||
def test_get_model_cost_map_stamps_loaded_at():
|
||||
"""The load time feeds each pod's reload-due decision; a load that does not stamp it
|
||||
would make manual reload requests race the proxy's startup"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from litellm.litellm_core_utils import get_model_cost_map as module
|
||||
|
||||
monkeypatch.setattr(module._cost_map_source_info, "loaded_at", None)
|
||||
client, _calls = _mock_client(
|
||||
[httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client
|
||||
)
|
||||
|
|
@ -323,6 +334,7 @@ def test_get_model_cost_map_stamps_loaded_at(monkeypatch):
|
|||
|
||||
import functools
|
||||
import random
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -500,6 +512,67 @@ async def test_refetch_respects_local_env_override(monkeypatch):
|
|||
assert len(result.model_cost_map) > 100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refetch_records_the_blob_id_of_the_bytes_served_and_the_fetch_etag():
|
||||
body = _real_map_bytes()
|
||||
client, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"abc123"'}, content=body)])
|
||||
|
||||
result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client)
|
||||
|
||||
assert isinstance(result, ModelCostMapReloaded)
|
||||
assert result.revision == git_blob_id(body)
|
||||
assert result.etag == 'W/"abc123"'
|
||||
assert get_model_cost_map_provenance() == {"source_revision": git_blob_id(body), "etag": 'W/"abc123"'}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refetch_revision_follows_the_bytes_not_the_url():
|
||||
edited = json.loads(_real_map_bytes())
|
||||
edited["gpt-5.4-mini"]["input_cost_per_token"] = 0.5
|
||||
client, _ = _mock_client(
|
||||
[httpx.Response(200, content=_real_map_bytes()), httpx.Response(200, content=json.dumps(edited).encode())]
|
||||
)
|
||||
|
||||
first = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client)
|
||||
second = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client)
|
||||
|
||||
assert isinstance(first, ModelCostMapReloaded) and isinstance(second, ModelCostMapReloaded)
|
||||
assert first.revision != second.revision
|
||||
assert get_model_cost_map_provenance()["source_revision"] == second.revision
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refetch_local_override_reports_the_bundled_blob_id_without_an_etag(monkeypatch):
|
||||
remote, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"remote"'}, content=_real_map_bytes())])
|
||||
await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=remote)
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
|
||||
result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0))
|
||||
|
||||
assert isinstance(result, ModelCostMapReloaded)
|
||||
assert result.revision == _bundled_blob_id()
|
||||
assert get_model_cost_map_provenance() == {"source_revision": _bundled_blob_id(), "etag": None}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refetch_stamps_loaded_at_on_remote_and_local_reloads(monkeypatch):
|
||||
from litellm.litellm_core_utils import get_model_cost_map as module
|
||||
|
||||
client, _ = _mock_client([httpx.Response(200, content=_real_map_bytes())])
|
||||
before_remote = datetime.now(timezone.utc)
|
||||
await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client)
|
||||
remote_loaded_at = module.get_model_cost_map_loaded_at()
|
||||
assert remote_loaded_at is not None
|
||||
assert before_remote <= remote_loaded_at <= datetime.now(timezone.utc)
|
||||
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
before_local = datetime.now(timezone.utc)
|
||||
await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0))
|
||||
local_loaded_at = module.get_model_cost_map_loaded_at()
|
||||
assert local_loaded_at is not None
|
||||
assert before_local <= local_loaded_at <= datetime.now(timezone.utc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_model_cost_map: the boot-time load retries transient failures like a reload does
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -592,3 +665,49 @@ def test_boot_load_respects_local_env_override(monkeypatch):
|
|||
)
|
||||
assert len(cost_map) > 100
|
||||
assert get_model_cost_map_source_info()["is_env_forced"] is True
|
||||
|
||||
|
||||
def test_boot_load_records_the_blob_id_of_the_bytes_served_and_the_fetch_etag():
|
||||
body = _real_map_bytes()
|
||||
client, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=body)], client_cls=httpx.Client)
|
||||
|
||||
get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=client)
|
||||
|
||||
source = get_model_cost_map_source_info()
|
||||
assert source["source"] == "remote"
|
||||
assert source["etag"] == 'W/"boot"'
|
||||
assert source["source_revision"] == git_blob_id(body)
|
||||
assert source["loaded_at"] is not None
|
||||
|
||||
|
||||
def test_boot_load_fallback_to_the_backup_reports_its_blob_id_and_drops_the_remote_etag():
|
||||
remote, _ = _mock_client(
|
||||
[httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_real_map_bytes())], client_cls=httpx.Client
|
||||
)
|
||||
get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=remote)
|
||||
failing, _ = _mock_client([httpx.Response(404)], client_cls=httpx.Client)
|
||||
|
||||
get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=failing)
|
||||
|
||||
source = get_model_cost_map_source_info()
|
||||
assert source["source"] == "local"
|
||||
assert source["etag"] is None
|
||||
assert source["source_revision"] == _bundled_blob_id()
|
||||
|
||||
|
||||
def test_boot_load_that_fails_the_integrity_check_reports_the_backup_not_the_rejected_fetch():
|
||||
remote, _ = _mock_client(
|
||||
[httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_real_map_bytes())], client_cls=httpx.Client
|
||||
)
|
||||
get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=remote)
|
||||
shrunk_body = b'{"gpt-5.4-mini": {"mode": "chat", "input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06}}'
|
||||
shrunk, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"shrunk"'}, content=shrunk_body)], client_cls=httpx.Client)
|
||||
|
||||
get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=shrunk)
|
||||
|
||||
source = get_model_cost_map_source_info()
|
||||
assert source["source"] == "local"
|
||||
assert source["fallback_reason"] == "Remote data failed integrity validation"
|
||||
assert source["etag"] is None
|
||||
assert source["source_revision"] == _bundled_blob_id()
|
||||
assert source["source_revision"] != git_blob_id(shrunk_body)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import base64
|
||||
import json
|
||||
from typing import Any, Final, cast
|
||||
|
||||
import pytest
|
||||
|
|
@ -724,9 +725,14 @@ def test_translate_anthropic_to_openai_orders_top_level_and_midturn_system():
|
|||
]
|
||||
|
||||
|
||||
def _translate_with_metadata(
|
||||
model: str, metadata: dict[str, str], custom_llm_provider: str | None
|
||||
) -> dict[str, Any]:
|
||||
def _claude_code_user_id(session_id: str) -> str:
|
||||
return json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": session_id})
|
||||
|
||||
|
||||
CLAUDE_CODE_USER_ID: Final = _claude_code_user_id("session-abc")
|
||||
|
||||
|
||||
def _translate_with_metadata(model: str, metadata: dict[str, str], custom_llm_provider: str | None) -> dict[str, Any]:
|
||||
openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
|
||||
anthropic_message_request={
|
||||
"model": model,
|
||||
|
|
@ -739,23 +745,51 @@ def _translate_with_metadata(
|
|||
return cast(dict[str, Any], openai_request)
|
||||
|
||||
|
||||
def test_translate_anthropic_to_openai_maps_user_id_to_prompt_cache_key_for_openai():
|
||||
openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": "session-abc"}, "openai")
|
||||
assert openai_request["user"] == "session-abc"
|
||||
def test_translate_anthropic_to_openai_maps_claude_code_session_id_to_prompt_cache_key_for_openai():
|
||||
openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": CLAUDE_CODE_USER_ID}, "openai")
|
||||
assert openai_request["user"] == CLAUDE_CODE_USER_ID
|
||||
assert openai_request["prompt_cache_key"] == "session-abc"
|
||||
|
||||
|
||||
def test_translate_anthropic_to_openai_truncates_prompt_cache_key_but_keeps_full_user():
|
||||
long_id = "".join(str(i % 10) for i in range(100))
|
||||
openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": long_id}, "openai")
|
||||
assert openai_request["user"] == long_id
|
||||
assert openai_request["prompt_cache_key"] == long_id[:64]
|
||||
assert len(openai_request["prompt_cache_key"]) == 64
|
||||
def test_translate_anthropic_to_openai_gives_each_claude_code_session_its_own_prompt_cache_key():
|
||||
"""BerriAI/litellm#39145: the first 64 chars of Claude Code's user_id are the per-install device_id."""
|
||||
keys = tuple(
|
||||
_translate_with_metadata("openai/gpt-5.6-luna", {"user_id": _claude_code_user_id(session_id)}, "openai")[
|
||||
"prompt_cache_key"
|
||||
]
|
||||
for session_id in ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222")
|
||||
)
|
||||
assert keys == ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222")
|
||||
|
||||
|
||||
def test_translate_anthropic_to_openai_truncates_long_session_id_to_openai_limit():
|
||||
long_session_id = "".join(str(i % 10) for i in range(100))
|
||||
openai_request = _translate_with_metadata(
|
||||
"openai/gpt-5.6-luna", {"user_id": _claude_code_user_id(long_session_id)}, "openai"
|
||||
)
|
||||
assert openai_request["prompt_cache_key"] == long_session_id[:64]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"user_id",
|
||||
[
|
||||
"alice",
|
||||
"".join(str(i % 10) for i in range(100)),
|
||||
json.dumps({"device_id": "d" * 64, "account_uuid": ""}),
|
||||
json.dumps({"session_id": ""}),
|
||||
json.dumps({"session_id": 123}),
|
||||
"{not json",
|
||||
],
|
||||
)
|
||||
def test_translate_anthropic_to_openai_keeps_plain_user_id_off_prompt_cache_key(user_id: str):
|
||||
openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": user_id}, "openai")
|
||||
assert openai_request["user"] == user_id
|
||||
assert "prompt_cache_key" not in openai_request
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["azure/my-gpt-5-deployment", "my-gpt-5-deployment"])
|
||||
def test_translate_anthropic_to_openai_sets_prompt_cache_key_for_azure(model: str):
|
||||
openai_request = _translate_with_metadata(model, {"user_id": "session-abc"}, "azure")
|
||||
openai_request = _translate_with_metadata(model, {"user_id": CLAUDE_CODE_USER_ID}, "azure")
|
||||
assert openai_request["prompt_cache_key"] == "session-abc"
|
||||
|
||||
|
||||
|
|
@ -772,8 +806,8 @@ def test_translate_anthropic_to_openai_sets_prompt_cache_key_for_azure(model: st
|
|||
def test_translate_anthropic_to_openai_skips_prompt_cache_key_when_provider_lacks_it(
|
||||
model: str, custom_llm_provider: str
|
||||
):
|
||||
openai_request = _translate_with_metadata(model, {"user_id": "session-abc"}, custom_llm_provider)
|
||||
assert openai_request["user"] == "session-abc"
|
||||
openai_request = _translate_with_metadata(model, {"user_id": CLAUDE_CODE_USER_ID}, custom_llm_provider)
|
||||
assert openai_request["user"] == CLAUDE_CODE_USER_ID
|
||||
assert "prompt_cache_key" not in openai_request
|
||||
|
||||
|
||||
|
|
@ -781,14 +815,14 @@ def test_translate_anthropic_to_openai_skips_prompt_cache_key_for_chained_litell
|
|||
assert "prompt_cache_key" in litellm.get_supported_openai_params(
|
||||
model="xai", custom_llm_provider="litellm_proxy"
|
||||
)
|
||||
openai_request = _translate_with_metadata("litellm_proxy/xai", {"user_id": "session-abc"}, "litellm_proxy")
|
||||
assert openai_request["user"] == "session-abc"
|
||||
openai_request = _translate_with_metadata("litellm_proxy/xai", {"user_id": CLAUDE_CODE_USER_ID}, "litellm_proxy")
|
||||
assert openai_request["user"] == CLAUDE_CODE_USER_ID
|
||||
assert "prompt_cache_key" not in openai_request
|
||||
|
||||
|
||||
def test_translate_anthropic_to_openai_skips_prompt_cache_key_without_provider():
|
||||
openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": "session-abc"}, None)
|
||||
assert openai_request["user"] == "session-abc"
|
||||
openai_request = _translate_with_metadata("openai/gpt-5.6-luna", {"user_id": CLAUDE_CODE_USER_ID}, None)
|
||||
assert openai_request["user"] == CLAUDE_CODE_USER_ID
|
||||
assert "prompt_cache_key" not in openai_request
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
|
@ -10,6 +11,7 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.handler import (
|
|||
)
|
||||
|
||||
MESSAGES = [{"role": "user", "content": "hello"}]
|
||||
CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"})
|
||||
|
||||
|
||||
def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, object] | None = None):
|
||||
|
|
@ -17,7 +19,7 @@ def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, ob
|
|||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model=model,
|
||||
metadata={"user_id": "session-abc"},
|
||||
metadata={"user_id": CLAUDE_CODE_USER_ID},
|
||||
thinking=thinking,
|
||||
extra_kwargs=extra_kwargs,
|
||||
)
|
||||
|
|
@ -26,7 +28,7 @@ def _prepare(model: str, extra_kwargs: dict[str, object], thinking: dict[str, ob
|
|||
|
||||
def test_prepare_completion_kwargs_derives_prompt_cache_key_for_openai_provider():
|
||||
completion_kwargs = _prepare("openai/gpt-5.6-luna", {"custom_llm_provider": "openai"})
|
||||
assert completion_kwargs["user"] == "session-abc"
|
||||
assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID
|
||||
assert completion_kwargs["prompt_cache_key"] == "session-abc"
|
||||
|
||||
|
||||
|
|
@ -35,7 +37,7 @@ def test_prepare_completion_kwargs_prefers_explicit_prompt_cache_key_over_derive
|
|||
"openai/gpt-5.6-luna",
|
||||
{"custom_llm_provider": "openai", "prompt_cache_key": "explicit-key"},
|
||||
)
|
||||
assert completion_kwargs["user"] == "session-abc"
|
||||
assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID
|
||||
assert completion_kwargs["prompt_cache_key"] == "explicit-key"
|
||||
|
||||
|
||||
|
|
@ -50,13 +52,13 @@ def test_prepare_completion_kwargs_skips_prompt_cache_key_without_provider_suppo
|
|||
model: str, extra_kwargs: dict[str, object]
|
||||
):
|
||||
completion_kwargs = _prepare(model, extra_kwargs)
|
||||
assert completion_kwargs["user"] == "session-abc"
|
||||
assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID
|
||||
assert "prompt_cache_key" not in completion_kwargs
|
||||
|
||||
|
||||
def test_prepare_completion_kwargs_skips_prompt_cache_key_for_chained_litellm_proxy():
|
||||
completion_kwargs = _prepare("litellm_proxy/xai", {"custom_llm_provider": "litellm_proxy"})
|
||||
assert completion_kwargs["user"] == "session-abc"
|
||||
assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID
|
||||
assert "prompt_cache_key" not in completion_kwargs
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler
|
|||
)
|
||||
|
||||
MESSAGES = [{"role": "user", "content": "hello"}]
|
||||
CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"})
|
||||
|
||||
RESPONSES_SSE_BODY = (
|
||||
b"event: response.created\n"
|
||||
|
|
@ -30,7 +31,19 @@ RESPONSES_SSE_BODY = (
|
|||
)
|
||||
|
||||
|
||||
def test_build_responses_kwargs_derives_prompt_cache_key_from_user_id():
|
||||
def test_build_responses_kwargs_derives_prompt_cache_key_from_claude_code_session_id():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
metadata={"user_id": CLAUDE_CODE_USER_ID},
|
||||
extra_kwargs={"custom_llm_provider": "openai"},
|
||||
)
|
||||
assert responses_kwargs["user"] == CLAUDE_CODE_USER_ID[:64]
|
||||
assert responses_kwargs["prompt_cache_key"] == "session-abc"
|
||||
|
||||
|
||||
def test_build_responses_kwargs_sets_no_prompt_cache_key_for_plain_user_id():
|
||||
responses_kwargs = _build_responses_kwargs(
|
||||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
|
|
@ -39,7 +52,7 @@ def test_build_responses_kwargs_derives_prompt_cache_key_from_user_id():
|
|||
extra_kwargs={"custom_llm_provider": "openai"},
|
||||
)
|
||||
assert responses_kwargs["user"] == "session-abc"
|
||||
assert responses_kwargs["prompt_cache_key"] == "session-abc"
|
||||
assert "prompt_cache_key" not in responses_kwargs
|
||||
|
||||
|
||||
def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived():
|
||||
|
|
@ -47,10 +60,10 @@ def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived()
|
|||
max_tokens=1024,
|
||||
messages=MESSAGES,
|
||||
model="openai/gpt-5.6-luna",
|
||||
metadata={"user_id": "session-abc"},
|
||||
metadata={"user_id": CLAUDE_CODE_USER_ID},
|
||||
extra_kwargs={"custom_llm_provider": "openai", "prompt_cache_key": "explicit-key"},
|
||||
)
|
||||
assert responses_kwargs["user"] == "session-abc"
|
||||
assert responses_kwargs["user"] == CLAUDE_CODE_USER_ID[:64]
|
||||
assert responses_kwargs["prompt_cache_key"] == "explicit-key"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1113,17 +1113,34 @@ class TestTranslateRequestBroaderCoverage:
|
|||
kwargs = _ADAPTER.translate_request(req)
|
||||
assert len(kwargs["user"]) == 64
|
||||
|
||||
def test_metadata_user_id_mapped_to_prompt_cache_key(self):
|
||||
req = _make_request(metadata={"user_id": "user-42"})
|
||||
def test_metadata_claude_code_session_id_mapped_to_prompt_cache_key(self):
|
||||
user_id = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-42"})
|
||||
req = _make_request(metadata={"user_id": user_id})
|
||||
kwargs = _ADAPTER.translate_request(req)
|
||||
assert kwargs["prompt_cache_key"] == "user-42"
|
||||
assert kwargs["user"] == user_id[:64]
|
||||
assert kwargs["prompt_cache_key"] == "session-42"
|
||||
|
||||
def test_metadata_user_id_prompt_cache_key_truncated_to_first_64_chars(self):
|
||||
long_id = "".join(str(i % 10) for i in range(100))
|
||||
req = _make_request(metadata={"user_id": long_id})
|
||||
def test_metadata_claude_code_sessions_get_distinct_prompt_cache_keys(self):
|
||||
"""BerriAI/litellm#39145: the first 64 chars of Claude Code's user_id are the per-install device_id."""
|
||||
keys = tuple(
|
||||
_ADAPTER.translate_request(
|
||||
_make_request(
|
||||
metadata={"user_id": json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": sid})}
|
||||
)
|
||||
)["prompt_cache_key"]
|
||||
for sid in ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222")
|
||||
)
|
||||
assert keys == ("11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"user_id",
|
||||
["user-42", "".join(str(i % 10) for i in range(100)), json.dumps({"device_id": "d" * 64}), "{not json"],
|
||||
)
|
||||
def test_metadata_plain_user_id_sets_no_prompt_cache_key(self, user_id: str):
|
||||
req = _make_request(metadata={"user_id": user_id})
|
||||
kwargs = _ADAPTER.translate_request(req)
|
||||
assert kwargs["prompt_cache_key"] == long_id[:64]
|
||||
assert len(kwargs["prompt_cache_key"]) == 64
|
||||
assert kwargs["user"] == user_id[:64]
|
||||
assert "prompt_cache_key" not in kwargs
|
||||
|
||||
def test_metadata_empty_user_id_sets_no_prompt_cache_key(self):
|
||||
req = _make_request(metadata={"user_id": ""})
|
||||
|
|
|
|||
|
|
@ -2,15 +2,15 @@ import os
|
|||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
|
||||
global_mcp_server_manager,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _hermetic_mcp_server_registry():
|
||||
"""Restore the singleton ``global_mcp_server_manager``'s registry state around every
|
||||
test, so entries seeded by one test never leak into another on a shared shard."""
|
||||
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
|
||||
global_mcp_server_manager,
|
||||
)
|
||||
|
||||
saved_registry = dict(global_mcp_server_manager.registry)
|
||||
saved_config_servers = dict(global_mcp_server_manager.config_mcp_servers)
|
||||
saved_tool_mapping = dict(global_mcp_server_manager.tool_name_to_mcp_server_name_mapping)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from mcp.types import CallToolResult, TextContent, Tool
|
|||
|
||||
from litellm.proxy._experimental.mcp_server import server
|
||||
from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing
|
||||
from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import (
|
||||
MCP_PROXY_CALL_TOOL_NAME,
|
||||
MCP_PROXY_SCHEMA_TOOL_NAME,
|
||||
|
|
@ -271,6 +270,8 @@ class TestMcpProxyAuthorizationScope:
|
|||
|
||||
@pytest.fixture
|
||||
def rig(self) -> Iterator[AsyncMock]:
|
||||
from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
|
||||
|
||||
upstream = {
|
||||
"srv-alpha": [_upstream_tool("alpha", "add"), _upstream_tool("alpha", "multiply")],
|
||||
"srv-beta": [_upstream_tool("beta", "add")],
|
||||
|
|
|
|||
|
|
@ -1,8 +1,13 @@
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import ANY, AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -7005,3 +7010,126 @@ class TestLitellmReceivedAtStamping:
|
|||
|
||||
assert result == earlier
|
||||
assert request.state.litellm_received_at == earlier
|
||||
|
||||
|
||||
_RECORDING_DDTRACE = dedent(
|
||||
'''
|
||||
import functools
|
||||
import inspect
|
||||
|
||||
|
||||
class _Span:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return None
|
||||
|
||||
|
||||
class _Tracer:
|
||||
def __init__(self):
|
||||
self.spans = []
|
||||
|
||||
def wrap(self, name=None, **kwargs):
|
||||
def decorator(f):
|
||||
span_name = name or f"{f.__module__}.{f.__name__}"
|
||||
if inspect.iscoroutinefunction(f):
|
||||
|
||||
@functools.wraps(f)
|
||||
async def async_wrapped(*args, **kw):
|
||||
self.spans.append(span_name)
|
||||
return await f(*args, **kw)
|
||||
|
||||
return async_wrapped
|
||||
|
||||
@functools.wraps(f)
|
||||
def wrapped(*args, **kw):
|
||||
self.spans.append(span_name)
|
||||
return f(*args, **kw)
|
||||
|
||||
return wrapped
|
||||
|
||||
return decorator
|
||||
|
||||
def trace(self, name, **kwargs):
|
||||
return _Span()
|
||||
|
||||
def current_span(self):
|
||||
return None
|
||||
|
||||
def current_root_span(self):
|
||||
return None
|
||||
|
||||
|
||||
tracer = _Tracer()
|
||||
'''
|
||||
)
|
||||
|
||||
_DDTRACE_AUTH_PROBE = dedent(
|
||||
'''
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import ddtrace
|
||||
from fastapi import Request
|
||||
from starlette.datastructures import URL
|
||||
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
||||
proxy_server.master_key = "sk-probe"
|
||||
|
||||
|
||||
async def auth(api_key):
|
||||
request = Request(scope={"type": "http", "headers": [], "method": "POST", "path": "/chat/completions"})
|
||||
request._url = URL(url="/chat/completions")
|
||||
try:
|
||||
await user_api_key_auth(
|
||||
request=request,
|
||||
api_key=api_key,
|
||||
azure_api_key_header="",
|
||||
anthropic_api_key_header=None,
|
||||
google_ai_studio_api_key_header=None,
|
||||
azure_apim_header=None,
|
||||
custom_litellm_key_header=None,
|
||||
)
|
||||
return "accepted"
|
||||
except ProxyException:
|
||||
return "rejected"
|
||||
|
||||
|
||||
async def main():
|
||||
outcomes = [await auth("Bearer sk-probe"), await auth("Bearer sk-wrong")]
|
||||
print(json.dumps({"outcomes": outcomes, "spans": ddtrace.tracer.spans}))
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
'''
|
||||
)
|
||||
|
||||
|
||||
def test_user_api_key_auth_opens_a_datadog_span_for_accepted_and_rejected_keys(tmp_path: Path):
|
||||
stub_root = tmp_path / "site"
|
||||
(stub_root / "ddtrace").mkdir(parents=True)
|
||||
(stub_root / "ddtrace" / "__init__.py").write_text(_RECORDING_DDTRACE)
|
||||
probe = tmp_path / "probe.py"
|
||||
probe.write_text(_DDTRACE_AUTH_PROBE)
|
||||
repo_root = Path(litellm.__file__).resolve().parent.parent
|
||||
env = {
|
||||
**os.environ,
|
||||
"USE_DDTRACE": "true",
|
||||
"PYTHONPATH": os.pathsep.join(
|
||||
[str(stub_root), str(repo_root)] + [p for p in (os.environ.get("PYTHONPATH"),) if p]
|
||||
),
|
||||
}
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(probe)], env=env, cwd=repo_root, capture_output=True, text=True, check=False
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr[-4000:]
|
||||
report = json.loads(result.stdout.strip().splitlines()[-1])
|
||||
assert report["outcomes"] == ["accepted", "rejected"]
|
||||
auth_span = "litellm.proxy.auth.user_api_key_auth.user_api_key_auth"
|
||||
assert [span for span in report["spans"] if span == auth_span] == [auth_span, auth_span]
|
||||
|
|
|
|||
|
|
@ -11,15 +11,21 @@ Routes covered:
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
|
||||
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map_provenance
|
||||
|
||||
from .conftest import VOLATILE_KEYS, normalize
|
||||
|
||||
# Some response bodies include a "timestamp" — extend the volatile set so
|
||||
# dict-equality assertions remain stable.
|
||||
_VOLATILE = VOLATILE_KEYS | frozenset({"timestamp"})
|
||||
|
||||
_SERVED_ETAG = 'W/"cost-map-etag"'
|
||||
_ROOT_COST_MAP = Path(__file__).resolve().parents[4] / "model_prices_and_context_window.json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
|
|
@ -83,6 +89,7 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
|
|||
"status": "success",
|
||||
"models_count": 2,
|
||||
"timestamp": "<VOLATILE>",
|
||||
**get_model_cost_map_provenance(),
|
||||
}
|
||||
assert table.upsert.await_count == 1
|
||||
update_payload = table.upsert.await_args.kwargs["data"]["update"]
|
||||
|
|
@ -90,6 +97,54 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
|
|||
assert update_payload["reload_revision"] == {"increment": 1}
|
||||
|
||||
|
||||
def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every_status_surface(
|
||||
client, auth_as, monkeypatch, mock_prisma
|
||||
):
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import git_blob_id
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
_attach_litellm_config(mock_prisma)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False)
|
||||
body = _ROOT_COST_MAP.read_bytes()
|
||||
expected = {"source_revision": git_blob_id(body), "etag": _SERVED_ETAG}
|
||||
served = httpx.Response(200, headers={"ETag": _SERVED_ETAG}, content=body)
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.get_model_cost_map._default_reload_client",
|
||||
lambda: httpx.AsyncClient(transport=httpx.MockTransport(lambda request: served)),
|
||||
)
|
||||
monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None)
|
||||
monkeypatch.setattr("litellm.model_cost", {}, raising=False)
|
||||
|
||||
async def _fake_invalidate(name):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(ps, "invalidate_config_param", _fake_invalidate)
|
||||
|
||||
with auth_as(LitellmUserRoles.PROXY_ADMIN):
|
||||
reload_response = client.post("/reload/model_cost_map")
|
||||
source_response = client.get("/model/cost_map/source")
|
||||
status_response = client.get("/schedule/model_cost_map_reload/status")
|
||||
public_response = client.get("/public/litellm_model_cost_map")
|
||||
|
||||
assert reload_response.status_code == 200
|
||||
reload_body = reload_response.json()
|
||||
assert {key: reload_body[key] for key in expected} == expected
|
||||
assert source_response.status_code == 200
|
||||
source_body = source_response.json()
|
||||
assert {key: source_body[key] for key in expected} == expected
|
||||
assert source_body["source"] == "remote"
|
||||
assert status_response.status_code == 200
|
||||
assert {key: status_response.json()[key] for key in expected} == expected
|
||||
assert public_response.status_code == 200
|
||||
assert "gpt-4o" in public_response.json()
|
||||
assert reload_body["models_count"] == len(litellm.model_cost)
|
||||
|
||||
|
||||
def test_reload_model_cost_map_fetch_failure_502_keeps_map(
|
||||
client, auth_as, monkeypatch, mock_prisma
|
||||
):
|
||||
|
|
@ -270,7 +325,7 @@ def test_cancel_model_cost_map_reload_no_db_500(client, auth_as, monkeypatch):
|
|||
def test_get_model_cost_map_reload_status_no_db_not_scheduled(
|
||||
client, auth_as, monkeypatch
|
||||
):
|
||||
"""No prisma client → returns the not-scheduled shape (4 keys, all-null)."""
|
||||
"""No prisma client → returns the not-scheduled shape (all-null) plus the cost map provenance."""
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
|
|
@ -283,6 +338,7 @@ def test_get_model_cost_map_reload_status_no_db_not_scheduled(
|
|||
"interval_hours": None,
|
||||
"last_run": None,
|
||||
"next_run": None,
|
||||
**get_model_cost_map_provenance(),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -309,6 +365,7 @@ def test_get_model_cost_map_reload_status_scheduled(
|
|||
"interval_hours": 12,
|
||||
"last_run": None,
|
||||
"next_run": None,
|
||||
**get_model_cost_map_provenance(),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -337,6 +394,7 @@ def test_get_model_cost_map_reload_status_reports_persisted_last_run(
|
|||
"interval_hours": 6,
|
||||
"last_run": "2024-01-01T06:00:00+00:00",
|
||||
"next_run": "2024-01-01T12:00:00+00:00",
|
||||
**get_model_cost_map_provenance(),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -365,6 +423,7 @@ def test_get_model_cost_map_reload_status_no_config_not_scheduled(
|
|||
"interval_hours": None,
|
||||
"last_run": None,
|
||||
"next_run": None,
|
||||
**get_model_cost_map_provenance(),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -391,6 +450,8 @@ def test_get_model_cost_map_source_happy(client, auth_as, monkeypatch):
|
|||
"url": "https://example.invalid/cost_map.json",
|
||||
"is_env_forced": False,
|
||||
"fallback_reason": None,
|
||||
"loaded_at": "2026-09-07T01:02:03+00:00",
|
||||
**get_model_cost_map_provenance(),
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.get_model_cost_map_source_info",
|
||||
|
|
@ -406,6 +467,8 @@ def test_get_model_cost_map_source_happy(client, auth_as, monkeypatch):
|
|||
"url": "https://example.invalid/cost_map.json",
|
||||
"is_env_forced": False,
|
||||
"fallback_reason": None,
|
||||
"loaded_at": "2026-09-07T01:02:03+00:00",
|
||||
**get_model_cost_map_provenance(),
|
||||
"model_count": 3,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -32,8 +32,16 @@ const remoteSource = {
|
|||
url: "https://pricing.example.test/model_prices.json",
|
||||
is_env_forced: false,
|
||||
fallback_reason: null,
|
||||
loaded_at: null,
|
||||
source_revision: null,
|
||||
etag: null,
|
||||
model_count: 1234,
|
||||
};
|
||||
const provenance = {
|
||||
loaded_at: "2026-09-07T10:00:00Z",
|
||||
source_revision: "4273ec544726bf255ea920533e209e6022653bb4",
|
||||
etag: 'W/"eb8e9a53f4cc284b"',
|
||||
};
|
||||
|
||||
describe("PriceDataReload", () => {
|
||||
beforeEach(() => {
|
||||
|
|
@ -51,6 +59,43 @@ describe("PriceDataReload", () => {
|
|||
expect(screen.getByText("No periodic reload scheduled")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows which revision of the cost map is loaded when the source reports one", async () => {
|
||||
vi.mocked(getModelCostMapSource).mockResolvedValue({ ...remoteSource, ...provenance } as never);
|
||||
render(<PriceDataReload accessToken="sk-test" />);
|
||||
|
||||
expect(await screen.findByText("Source revision:")).toBeInTheDocument();
|
||||
expect(screen.getByText("4273ec544726")).toBeInTheDocument();
|
||||
expect(screen.getByText("ETag:")).toBeInTheDocument();
|
||||
expect(screen.getByText('W/"eb8e9a53f4cc284b"')).toBeInTheDocument();
|
||||
expect(screen.getByText("Loaded at:")).toBeInTheDocument();
|
||||
expect(screen.getByText(/worker that answered this request/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/Last run time is the latest reload any worker recorded/)).toBeInTheDocument();
|
||||
expect(screen.getByText(new Date(provenance.loaded_at).toLocaleString())).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows a malformed loaded_at as-is instead of Invalid Date", async () => {
|
||||
vi.mocked(getModelCostMapSource).mockResolvedValue({
|
||||
...remoteSource,
|
||||
...provenance,
|
||||
loaded_at: "yesterday-ish",
|
||||
} as never);
|
||||
render(<PriceDataReload accessToken="sk-test" />);
|
||||
|
||||
expect(await screen.findByText("Loaded at:")).toBeInTheDocument();
|
||||
expect(screen.getByText("yesterday-ish")).toBeInTheDocument();
|
||||
expect(screen.queryByText("Invalid Date")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("hides the provenance rows when the loaded map carries no stamp", async () => {
|
||||
render(<PriceDataReload accessToken="sk-test" />);
|
||||
|
||||
expect(await screen.findByText("Pricing Data Source")).toBeInTheDocument();
|
||||
expect(screen.queryByText("Source revision:")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("ETag:")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Loaded at:")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/worker that answered this request/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("confirms an immediate reload and refreshes dependent data", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onReloadSuccess = vi.fn();
|
||||
|
|
|
|||
|
|
@ -49,9 +49,16 @@ interface CostMapSourceInfo {
|
|||
url: string | null;
|
||||
is_env_forced: boolean;
|
||||
fallback_reason: string | null;
|
||||
loaded_at: string | null;
|
||||
source_revision: string | null;
|
||||
etag: string | null;
|
||||
model_count: number;
|
||||
}
|
||||
|
||||
const SHORT_REVISION_LENGTH = 12;
|
||||
|
||||
const shortRevision = (revision: string) => revision.slice(0, SHORT_REVISION_LENGTH);
|
||||
|
||||
const EMPTY_RELOAD_STATUS: ReloadStatus = {
|
||||
scheduled: false,
|
||||
interval_hours: null,
|
||||
|
|
@ -89,6 +96,55 @@ const isValidReloadInterval = (value: number) => {
|
|||
return value >= 1 && value <= 168;
|
||||
};
|
||||
|
||||
const formatDateTime = (dateTimeString: string | null) => {
|
||||
if (!dateTimeString) return "Never";
|
||||
const parsed = new Date(dateTimeString);
|
||||
return Number.isNaN(parsed.getTime()) ? dateTimeString : parsed.toLocaleString();
|
||||
};
|
||||
|
||||
const CostMapProvenanceRows: React.FC<{ sourceInfo: CostMapSourceInfo }> = ({ sourceInfo }) => (
|
||||
<>
|
||||
{sourceInfo.source_revision && (
|
||||
<div className="flex items-center justify-between gap-2 text-xs">
|
||||
<span className="text-muted-foreground">Source revision:</span>
|
||||
<Tooltip>
|
||||
<TooltipTrigger render={<code className="font-mono" />}>
|
||||
{shortRevision(sourceInfo.source_revision)}
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{sourceInfo.source_revision}</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{sourceInfo.etag && (
|
||||
<div className="flex items-center justify-between gap-2 text-xs">
|
||||
<span className="text-muted-foreground">ETag:</span>
|
||||
<Tooltip>
|
||||
<TooltipTrigger render={<code className="max-w-60 truncate font-mono" />}>{sourceInfo.etag}</TooltipTrigger>
|
||||
<TooltipContent>{sourceInfo.etag}</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{sourceInfo.loaded_at && (
|
||||
<div className="flex items-center justify-between text-xs">
|
||||
<span className="text-muted-foreground">Loaded at:</span>
|
||||
<span className="font-medium">{formatDateTime(sourceInfo.loaded_at)}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{sourceInfo.loaded_at && (
|
||||
<div className="flex items-center gap-1.5 text-xs text-muted-foreground">
|
||||
<Info className="size-3.5 shrink-0" />
|
||||
<span>
|
||||
Reported by the worker that answered this request. Other workers pick up a reload on their next poll, and the
|
||||
Last run time is the latest reload any worker recorded
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
|
||||
const PriceDataReload: React.FC<PriceDataReloadProps> = ({
|
||||
accessToken,
|
||||
onReloadSuccess,
|
||||
|
|
@ -227,15 +283,6 @@ const PriceDataReload: React.FC<PriceDataReloadProps> = ({
|
|||
}
|
||||
};
|
||||
|
||||
const formatDateTime = (dateTimeString: string | null) => {
|
||||
if (!dateTimeString) return "Never";
|
||||
try {
|
||||
return new Date(dateTimeString).toLocaleString();
|
||||
} catch {
|
||||
return dateTimeString;
|
||||
}
|
||||
};
|
||||
|
||||
const getStatusText = () => {
|
||||
if (!reloadStatus?.scheduled) return "Not scheduled";
|
||||
if (!reloadStatus.last_run) return "Ready";
|
||||
|
|
@ -334,6 +381,8 @@ const PriceDataReload: React.FC<PriceDataReloadProps> = ({
|
|||
</div>
|
||||
)}
|
||||
|
||||
<CostMapProvenanceRows sourceInfo={sourceInfo} />
|
||||
|
||||
{sourceInfo.is_env_forced && (
|
||||
<div className="flex items-center gap-1.5 text-xs text-muted-foreground">
|
||||
<Info className="size-3.5 shrink-0" />
|
||||
|
|
|
|||
|
|
@ -149,13 +149,12 @@ describe("autorouter_presets", () => {
|
|||
);
|
||||
});
|
||||
|
||||
// Opus serves both tiers, so the effort is all that separates them and losing it fails silently.
|
||||
it("pins the anthropic preset's reasoning tier to Opus at high thinking", () => {
|
||||
it("pins the anthropic preset's reasoning tier to Fable 5.1 at high thinking", () => {
|
||||
const config = getPresetByKey("anthropic_family")!.complexity_router_config;
|
||||
expect(config.tiers.COMPLEX).toEqual(["claude-opus-5"]);
|
||||
expect(config.tiers.REASONING).toEqual(["claude-opus-5"]);
|
||||
expect(config.tiers.REASONING).toEqual(["claude-fable-5-1"]);
|
||||
expect(config.tier_model_configs).toEqual({
|
||||
REASONING: [{ model_name: "claude-opus-5", litellm_params: { reasoning_effort: "high" } }],
|
||||
REASONING: [{ model_name: "claude-fable-5-1", litellm_params: { reasoning_effort: "high" } }],
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -214,7 +213,7 @@ describe("autorouter_presets", () => {
|
|||
const preset = getPresetByKey("anthropic_family")!;
|
||||
const prefill = buildPresetPrefill(preset.complexity_router_config, groupsOnly(getRequiredModelsInPreset(preset)));
|
||||
expect(prefill.complexityRouterConfig.tier_model_params).toEqual({
|
||||
REASONING: { "claude-opus-5": { reasoning_effort: "high" } },
|
||||
REASONING: { "claude-fable-5-1": { reasoning_effort: "high" } },
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -227,21 +226,21 @@ describe("autorouter_presets", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("pins the OpenAI preset to the Luna, Terra, and Sol progression", () => {
|
||||
it("pins the OpenAI preset to the Luna, Terra, Sol, and Astra progression", () => {
|
||||
const preset = getPresetByKey("openai_family")!;
|
||||
const expectedTiers = {
|
||||
SIMPLE: ["gpt-5.6-luna"],
|
||||
MEDIUM: ["gpt-5.6-terra"],
|
||||
COMPLEX: ["gpt-5.6-sol"],
|
||||
REASONING: ["gpt-5.6-sol"],
|
||||
REASONING: ["gpt-6-astra"],
|
||||
};
|
||||
expect(preset.complexity_router_config.tiers).toEqual(expectedTiers);
|
||||
expect(preset.complexity_router_config.tier_model_configs).toEqual({
|
||||
REASONING: [{ model_name: "gpt-5.6-sol", litellm_params: { reasoning_effort: "xhigh" } }],
|
||||
REASONING: [{ model_name: "gpt-6-astra", litellm_params: { reasoning_effort: "xhigh" } }],
|
||||
});
|
||||
const prefill = buildPresetPrefill(preset.complexity_router_config, groupsOnly(getRequiredModelsInPreset(preset)));
|
||||
expect(prefill.complexityRouterConfig.tier_model_params).toEqual({
|
||||
REASONING: { "gpt-5.6-sol": { reasoning_effort: "xhigh" } },
|
||||
REASONING: { "gpt-6-astra": { reasoning_effort: "xhigh" } },
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
3
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
3
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -8745,6 +8745,9 @@ export interface paths {
|
|||
* - url: the remote URL that was attempted (null when env-forced local)
|
||||
* - is_env_forced: true if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage
|
||||
* - fallback_reason: human-readable reason why remote failed (null on success)
|
||||
* - loaded_at: when this pod last loaded the map
|
||||
* - source_revision: git blob id of the loaded file, what git rev-parse <commit>:<path> prints for it
|
||||
* - etag: the ETag of the remote fetch (null for the bundled backup)
|
||||
* - model_count: number of models in the currently loaded cost map
|
||||
*/
|
||||
get: operations["get_model_cost_map_source_model_cost_map_source_get"];
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue