diff --git a/litellm/_logging.py b/litellm/_logging.py index 873a6619a81..5ba0c080364 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -3,6 +3,7 @@ import contextvars import functools import logging import os +import re import sys from datetime import datetime from logging import Formatter @@ -13,10 +14,11 @@ import litellm from litellm.constants import ( LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_STDOUT_SAFEGUARD_NOTE, + MAX_BASE64_LENGTH_STDOUT_LOG, MAX_STRING_LENGTH_STDOUT_LOG, ) from litellm.litellm_core_utils.env_utils import get_env_int -from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.secret_redaction import ( redact_internal_details, @@ -77,6 +79,37 @@ def _redact_structured_value(key: str | None, value: str) -> str: return redact_structured_value(key, value) +_REDACTED_RECORD_ATTR: Final = "litellm_redacted" +_REDACTED_STAMP: Final = object() +_UNREDACTED_SCALAR_TYPES: Final = (bool, int, float, type(None)) + + +def _is_redacted(record: logging.LogRecord) -> bool: + return getattr(record, _REDACTED_RECORD_ATTR, None) is _REDACTED_STAMP + + +def _scrubbing_changed_nothing(scrubbed: object, original: object) -> bool: + try: + return bool(scrubbed == original) + except Exception: + return False + + +def _plain_text(value: object) -> str: + try: + return str(value) + except Exception: + return UNSERIALIZABLE_OBJECT + + +def _redact_extra_value(key: str, value: object) -> object: + try: + scrubbed: Final = safe_json_structure(value, value_transform=_redact_structured_value, key=key) + except Exception: + return _redact_string(_plain_text(value)) + return value if _scrubbing_changed_nothing(scrubbed, value) else scrubbed + + def redact_secrets(value: str) -> str: """Public API: redact known secret/credential patterns from an arbitrary string. @@ -126,7 +159,7 @@ class SecretRedactionFilter(logging.Filter): _formatter = logging.Formatter() def filter(self, record: logging.LogRecord) -> bool: - if not _ENABLE_SECRET_REDACTION: + if not _ENABLE_SECRET_REDACTION or _is_redacted(record): return True # Runs before args are cleared, and before the extra-field loop below @@ -149,11 +182,19 @@ class SecretRedactionFilter(logging.Filter): except Exception: pass + if isinstance(record.stack_info, str): + record.stack_info = _redact_string(record.stack_info) # rebind-ok: a Filter scrubs records in place + # Redact extra fields passed via logger.debug("msg", extra={...}) for key, value in list(record.__dict__.items()): - if key not in _STANDARD_RECORD_ATTRS and isinstance(value, str): - setattr(record, key, _redact_string(value)) + if key in _STANDARD_RECORD_ATTRS: + continue + if isinstance(value, str): + setattr(record, key, _redact_structured_value(key, value)) + elif not isinstance(value, _UNREDACTED_SCALAR_TYPES): + setattr(record, key, _redact_extra_value(key, value)) + setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP) return True @@ -277,6 +318,51 @@ def _truncate_for_stdout_log(text: str, limit: int) -> str: return f"{text[:head_chars]}{_stdout_truncation_marker(len(text) - kept_chars)}{text[-tail_chars:]}" +_BYTES_PER_KIB: Final = 1024 +_BYTES_PER_MIB: Final = 1024 * 1024 + + +def format_base64_size(num_chars: int) -> str: + """Return a human-readable byte-size estimate from a base64 character count.""" + num_bytes: Final = num_chars * 3 / 4 + if num_bytes >= _BYTES_PER_MIB: + return f"{num_bytes / _BYTES_PER_MIB:.2f}MB" + if num_bytes >= _BYTES_PER_KIB: + return f"{num_bytes / _BYTES_PER_KIB:.1f}KB" + return f"{int(num_bytes)}B" + + +def _get_max_base64_length_stdout_log() -> int: + return get_env_int("MAX_BASE64_LENGTH_STDOUT_LOG", MAX_BASE64_LENGTH_STDOUT_LOG) + + +@functools.lru_cache(maxsize=8) +def _base64_run_pattern(min_chars: int) -> "re.Pattern[str]": + return re.compile(rf"(? bool: + unpadded: Final = run.rstrip("=") + is_hex_or_decimal: Final = not unpadded.strip(_LOWER_HEX_DIGITS) or not unpadded.strip(_UPPER_HEX_DIGITS) + is_one_repeated_char: Final = not unpadded.strip(unpadded[0]) + return not is_hex_or_decimal or is_one_repeated_char + + +def _replace_base64_run(match: "re.Match[str]") -> str: + run: Final = match.group(0) + if not _looks_like_base64(run): + return run + return f"[base64_data truncated: {format_base64_size(len(run))}]" + + +def _collapse_base64_runs(text: str, limit: int) -> str: + return _base64_run_pattern(limit + 1).sub(_replace_base64_run, text) + + class StdoutLogTruncationFilter(logging.Filter): """Bounds how much of an oversized log line reaches stdout. @@ -284,36 +370,42 @@ class StdoutLogTruncationFilter(logging.Filter): request writes hundreds of KB to stdout, repeatedly as the exception propagates from the router to the proxy handler and into its traceback, all inline on the event loop. - DEBUG records pass through untouched, since dumping full payloads is the point of + At every level, in the message and in the traceback alike, a base64 run longer than + MAX_BASE64_LENGTH_STDOUT_LOG collapses to a size placeholder first: a multi-megabyte + document upload otherwise costs seconds of event-loop time per DEBUG line in the + secret regex alone. Hex and decimal runs (digests, numeric ids) are left alone unless + they are one repeated character, which is what a zero-filled payload encodes to. + The text around a run stays, since dumping payloads is the point of `--detailed_debug`, and logging callbacks (OTEL, Datadog, etc.) don't run through - logging filters at all, so they still get the untruncated error. + logging filters at all, so they still get the untouched record. """ _formatter = logging.Formatter() def filter(self, record: logging.LogRecord) -> bool: - if record.levelno < logging.INFO: - return True - - limit: Final = _get_max_string_length_stdout_log() - if limit <= 0: - return True - try: message: Final = record.getMessage() except (TypeError, ValueError): return True - if len(message) > limit: - record.msg = _truncate_for_stdout_log(message, limit) # rebind-ok: the Filter interface mutates the record - record.args = None # rebind-ok: args are consumed by the truncated message above + base64_limit: Final = _get_max_base64_length_stdout_log() + collapsed: Final = _collapse_base64_runs(message, base64_limit) if base64_limit > 0 else message + limit: Final = _get_max_string_length_stdout_log() if record.levelno >= logging.INFO else 0 + bounded: Final = _truncate_for_stdout_log(collapsed, limit) if 0 < limit < len(collapsed) else collapsed + if bounded != message: + record.msg = bounded # rebind-ok: the Filter interface mutates the record + record.args = None # rebind-ok: args are consumed by the rewritten message above - if isinstance(record.exc_info, tuple): - exc_text: Final = record.exc_text or self._formatter.formatException(record.exc_info) - if len(exc_text) > limit: - record.exc_text = _truncate_for_stdout_log( # rebind-ok: the Filter interface mutates the record - exc_text, limit - ) + if not isinstance(record.exc_info, tuple): + return True + + exc_text: Final = record.exc_text or self._formatter.formatException(record.exc_info) + collapsed_exc: Final = _collapse_base64_runs(exc_text, base64_limit) if base64_limit > 0 else exc_text + bounded_exc: Final = ( + _truncate_for_stdout_log(collapsed_exc, limit) if 0 < limit < len(collapsed_exc) else collapsed_exc + ) + if bounded_exc != exc_text: + record.exc_text = bounded_exc # rebind-ok: the Filter interface mutates the record return True @@ -474,6 +566,7 @@ def _get_standard_record_attrs() -> frozenset: _STANDARD_RECORD_ATTRS: Final = _get_standard_record_attrs() +_NON_EXTRA_RECORD_ATTRS: Final = _STANDARD_RECORD_ATTRS | {_REDACTED_RECORD_ATTR} # CorrelationContextFilter is the only legitimate source for these two JSON fields; # see JsonFormatter.format() for why they're excluded from the generic message-content @@ -514,7 +607,7 @@ class JsonFormatter(Formatter): # Include extra attributes passed via logger.debug("msg", extra={...}) for key, value in record.__dict__.items(): - if key not in _STANDARD_RECORD_ATTRS and key not in json_record: + if key not in _NON_EXTRA_RECORD_ATTRS and key not in json_record: json_record[key] = value # trace_id/session_id are reserved: CorrelationContextFilter is the only @@ -538,7 +631,7 @@ class JsonFormatter(Formatter): if record.exc_info: json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info) - return safe_dumps(json_record, value_transform=_redact_structured_value) + return safe_dumps(json_record, value_transform=None if _is_redacted(record) else _redact_structured_value) class CorrelationPlainFormatter(logging.Formatter): @@ -549,7 +642,8 @@ class CorrelationPlainFormatter(logging.Formatter): """ def format(self, record: logging.LogRecord) -> str: - formatted: Final = _redact_string(super().format(record)) + rendered: Final = super().format(record) + formatted: Final = rendered if _is_redacted(record) else _redact_string(rendered) trace_id: Final = getattr(record, "trace_id", None) session_id: Final = getattr(record, "session_id", None) if not trace_id and not session_id: @@ -567,8 +661,8 @@ def _setup_json_exception_handlers(formatter): # Create a handler with JSON formatting for exceptions error_handler: Final = logging.StreamHandler() error_handler.setFormatter(formatter) - error_handler.addFilter(_secret_filter) error_handler.addFilter(_stdout_truncation_filter) + error_handler.addFilter(_secret_filter) error_handler.addFilter(_correlation_filter) # Setup excepthook for uncaught exceptions diff --git a/litellm/constants.py b/litellm/constants.py index 663af70c1c3..12749a0fce0 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -102,6 +102,7 @@ REDACTED_BY_LITELLM: Final = "redacted-by-litellm" REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}" MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096) +MAX_BASE64_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_BASE64_LENGTH_STDOUT_LOG", 4096) # When true, adds detailed per-phase timing breakdown headers to responses. # Headers: x-litellm-timing-{pre-processing,llm-api,post-processing,message-copy}-ms diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 3a1dbd24e86..b7067a45117 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -160,7 +160,7 @@ def get_llm_provider( if model is None: raise ValueError("model parameter is required but was None. Please provide a valid model name.") - if litellm.LiteLLMProxyChatConfig._should_use_litellm_proxy_by_default( + if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default( litellm_params=cast(LiteLLM_Params | None, litellm_params) ): return litellm.LiteLLMProxyChatConfig.litellm_proxy_get_custom_llm_provider_info( diff --git a/litellm/litellm_core_utils/logging_utils.py b/litellm/litellm_core_utils/logging_utils.py index 44daef42e14..0f14b461d3d 100644 --- a/litellm/litellm_core_utils/logging_utils.py +++ b/litellm/litellm_core_utils/logging_utils.py @@ -7,7 +7,7 @@ from collections.abc import Iterator, Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final -from litellm._logging import verbose_logger +from litellm._logging import format_base64_size, verbose_logger from litellm.constants import ( BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS, MAX_BASE64_LENGTH_FOR_LOGGING, @@ -40,9 +40,6 @@ import litellm Helper utils used for logging callbacks """ -_BYTES_PER_KIB: Final = 1024 -_BYTES_PER_MIB: Final = 1024 * 1024 - # Regex matching data-URI base64 content: "data:;base64," # Captures: group(1)=mime_type, group(2)=base64_payload _DATA_URI_RE: Final = re.compile(r"data:([^;]+);base64,([A-Za-z0-9+/=]+)") @@ -52,23 +49,13 @@ _DATA_URI_RE: Final = re.compile(r"data:([^;]+);base64,([A-Za-z0-9+/=]+)") _MAX_TRUNCATION_DEPTH: Final = 20 -def _format_base64_size(num_chars: int) -> str: - """Return a human-readable byte-size estimate from a base64 character count.""" - num_bytes: Final = num_chars * 3 / 4 - if num_bytes >= _BYTES_PER_MIB: - return f"{num_bytes / _BYTES_PER_MIB:.2f}MB" - if num_bytes >= _BYTES_PER_KIB: - return f"{num_bytes / _BYTES_PER_KIB:.1f}KB" - return f"{int(num_bytes)}B" - - def _base64_data_uri_replacer(match: re.Match) -> str: """Replace a single base64 data-URI match with a size placeholder if too long.""" mime_type: Final = match.group(1) payload: Final = match.group(2) if len(payload) <= MAX_BASE64_LENGTH_FOR_LOGGING: return match.group(0) - size_str: Final = _format_base64_size(len(payload)) + size_str: Final = format_base64_size(len(payload)) return f"data:{mime_type};base64,[base64_data truncated: {size_str}]" diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index 5b99e8cba98..4f9ac82d57d 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -6,25 +6,29 @@ from pydantic import BaseModel from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +UNSERIALIZABLE_OBJECT: Final = "Unserializable Object" + def strip_null_bytes(value: str) -> str: """Strip NUL bytes, which PostgreSQL text/jsonb columns reject (error 22P05).""" return value.replace("\x00", "") -def safe_dumps( - data: Any, +def safe_json_structure( + data: object, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH, value_transform: Callable[[str | None, str], str] | None = None, -) -> str: + key: str | None = None, +) -> object: """ - Recursively serialize data while detecting circular references. + Rebuild data out of JSON-native pieces while detecting circular references. If a circular reference is detected then a marker string is returned. NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. value_transform, when given, is applied to every string leaf (and to the str() fallback for non-serializable objects) with the mapping key the leaf was reached under, so callers can rewrite values without touching structure. + key is the mapping key data itself was reached under, when the caller has one. """ def _transform(key: str | None, value: str) -> str: @@ -75,7 +79,15 @@ def safe_dumps( try: return _transform(key, strip_null_bytes(str(obj))) except Exception: - return "Unserializable Object" + return UNSERIALIZABLE_OBJECT - safe_data: Final = _serialize(data, set(), 0) - return json.dumps(safe_data, default=str) + return _serialize(data, set(), 0, key) + + +def safe_dumps( + data: Any, + max_depth: int = DEFAULT_MAX_RECURSE_DEPTH, + value_transform: Callable[[str | None, str], str] | None = None, +) -> str: + """Serialize data to JSON text through safe_json_structure.""" + return json.dumps(safe_json_structure(data, max_depth, value_transform), default=str) diff --git a/litellm/llms/litellm_proxy/chat/transformation.py b/litellm/llms/litellm_proxy/chat/transformation.py index c11db6b000a..cf4c41cd3f3 100644 --- a/litellm/llms/litellm_proxy/chat/transformation.py +++ b/litellm/llms/litellm_proxy/chat/transformation.py @@ -54,7 +54,7 @@ class LiteLLMProxyChatConfig(OpenAIGPTConfig): return api_key or get_secret_str("LITELLM_PROXY_API_KEY") @staticmethod - def _should_use_litellm_proxy_by_default( + def should_use_litellm_proxy_by_default( litellm_params: LiteLLM_Params | None = None, ): """ diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py new file mode 100644 index 00000000000..cfe01e513fc --- /dev/null +++ b/litellm/llms/openai_like/model_info.py @@ -0,0 +1,92 @@ +import hashlib +import json +from collections.abc import Mapping +from types import MappingProxyType +from typing import Annotated, Final, TypeAlias + +import httpx +from pydantic import BaseModel, BeforeValidator, ConfigDict + +from litellm._logging import verbose_logger +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.utils import _add_path_to_api_base # pyright: ignore[reportPrivateUsage] # shared provider URL helper + +MODEL_INFO_REFRESH_SECONDS: Final = 300 +MODEL_INFO_REFRESH_CONCURRENCY: Final = 8 +MODEL_INFO_DISCOVERY_PROVIDERS: Final = frozenset({"hosted_vllm", "openai", "text-completion-openai", "openai_like"}) +_EMPTY_LIMITS: Final[Mapping[str, int]] = MappingProxyType({}) + + +def _positive_limit(value: object) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None + + +_TokenLimit: TypeAlias = Annotated[int | None, BeforeValidator(_positive_limit)] + + +class _ModelCard(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str + max_model_len: _TokenLimit = None + context_length: _TokenLimit = None + max_input_tokens: _TokenLimit = None + max_output_tokens: _TokenLimit = None + + def token_limits(self) -> Mapping[str, int]: + context: Final = self.max_model_len or self.context_length + input_limit: Final = self.max_input_tokens or context + output_limit: Final = self.max_output_tokens or context + return MappingProxyType( + { + key: value + for key, value in ( + ("max_tokens", context), + ("max_input_tokens", min(input_limit, context) if input_limit and context else input_limit), + ("max_output_tokens", min(output_limit, context) if output_limit and context else output_limit), + ) + if value is not None + } + ) + + +class _ModelList(BaseModel): + model_config = ConfigDict(frozen=True) + + data: tuple[_ModelCard, ...] = () + + +async def get_openai_compatible_model_info( + *, + model: str, + api_base: str, + headers: Mapping[str, str], + client: AsyncHTTPHandler, + cache: InMemoryCache, +) -> Mapping[str, int]: + url: Final = _add_path_to_api_base(api_base, "/v1/models") + cache_key: Final = ( + "upstream_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest() + ) + cached: Final[object] = cache.get_cache(cache_key) + if isinstance(cached, _ModelList): + return next((card.token_limits() for card in cached.data if card.id == model), _EMPTY_LIMITS) + + try: + response: Final = await client.get( + url=url, + headers=dict(headers), # mutable-ok: AsyncHTTPHandler requires a concrete dict + timeout=httpx.Timeout(5.0), + follow_redirects=False, + max_response_bytes=2 * 1024 * 1024, + ) + response.raise_for_status() + models: Final = _ModelList.model_validate_json(response.content) + except Exception: # noqa: BLE001 # optional upstream metadata must not interrupt proxy refresh + verbose_logger.debug("Could not discover upstream model token limits") + cache.set_cache(cache_key, _ModelList(), ttl=60) + return _EMPTY_LIMITS + + cache.set_cache(cache_key, models, ttl=MODEL_INFO_REFRESH_SECONDS) + return next((card.token_limits() for card in models.data if card.id == model), _EMPTY_LIMITS) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 599ae90bcae..c13b852484e 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -257,8 +257,8 @@ class DBSpendUpdateWriter: # Completion object fields kwargs: dict | None, completion_response: object, - start_time: datetime | None, - end_time: datetime | None, + start_time: datetime, + end_time: datetime, response_cost: float | None, ) -> bool: """Record the request's spend, answering whether its cost still needs charging. @@ -299,6 +299,7 @@ class DBSpendUpdateWriter: response_obj=completion_response, start_time=start_time, end_time=end_time, + llm_router=get_llm_router(), ) payload["spend"] = response_cost or 0.0 if isinstance(payload["startTime"], datetime): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2e8f7778a80..3af9aeccd69 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -305,6 +305,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( mask_sensitive_keys, ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import * @@ -1383,9 +1384,27 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: ## Initialize shared aiohttp session for connection reuse shared_aiohttp_session = await _initialize_shared_aiohttp_session() + model_info_scheduler: Final = scheduler if scheduler is not None else AsyncIOScheduler() + model_info_scheduler.add_job( + ProxyStartupEvent.refresh_model_info, + "interval", + seconds=MODEL_INFO_REFRESH_SECONDS, + id="refresh_model_info", + next_run_time=datetime.now(timezone.utc), + max_instances=1, + replace_existing=True, + ) + if not model_info_scheduler.running: + model_info_scheduler.start() + # End of startup event yield + if model_info_scheduler.running: + model_info_scheduler.remove_job("refresh_model_info") + if model_info_scheduler is not scheduler: + model_info_scheduler.shutdown(wait=False) + # Shutdown event - drain in-flight requests before tearing down dependencies # so SIGTERM (rolling update, scale-down, liveness kill) doesn't drop them. GracefulShutdownManager.start_shutdown() @@ -9344,6 +9363,11 @@ class ProxyStartupEvent: if isinstance(callback, _OPTIONAL_PromptInjectionDetection): callback.update_environment(router=llm_router) + @staticmethod + async def refresh_model_info() -> None: + if llm_router is not None: + await llm_router.arefresh_model_info() + @staticmethod def _warn_budget_without_db(max_budget: float | None, prisma_client: PrismaClient | None) -> None: if prisma_client is not None or not max_budget or max_budget <= 0: @@ -13600,8 +13624,11 @@ def _enrich_model_info_with_litellm_data( litellm_model_info = litellm.get_model_info(model=litellm_model, custom_llm_provider=split_model[0]) except Exception: litellm_model_info = {} - for k, v in litellm_model_info.items(): - if k not in model_info: + discovered_model_info: Final = ( + llm_router.get_discovered_model_info(model_info.get("id")) if llm_router is not None else MappingProxyType({}) + ) + for k, v in MappingProxyType({**litellm_model_info, **discovered_model_info}).items(): + if k not in model_info or (model_info[k] is None and k in discovered_model_info): model_info[k] = v model["model_info"] = model_info # don't return the api key / vertex credentials @@ -15066,8 +15093,11 @@ def _get_proxy_model_info(model: dict) -> dict: litellm_model_info = litellm.get_model_info(model=litellm_model, custom_llm_provider=split_model[0]) except Exception: litellm_model_info = {} - for k, v in litellm_model_info.items(): - if k not in model_info: + discovered_model_info: Final = ( + llm_router.get_discovered_model_info(model_info.get("id")) if llm_router is not None else MappingProxyType({}) + ) + for k, v in MappingProxyType({**litellm_model_info, **discovered_model_info}).items(): + if k not in model_info or (model_info[k] is None and k in discovered_model_info): model_info[k] = v model["model_info"] = model_info # don't return the llm credentials diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 56438fe45bd..52900c33745 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -5,7 +5,7 @@ from collections.abc import Mapping, Sequence from datetime import datetime, timezone from datetime import datetime as dt from types import MappingProxyType -from typing import Final, Literal, Protocol, cast, runtime_checkable +from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable from pydantic import BaseModel @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, reconstruct_model_name, ) +from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call from litellm.litellm_core_utils.litellm_logging import ( coerce_model_access_groups, @@ -43,10 +44,12 @@ from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload, SpendLogsR from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.proxy.utils import PrismaClient, hash_token +from litellm.types.router import DeploymentTypedDict, LiteLLM_Params from litellm.types.utils import ( PROMPT_CARRYING_GUARDRAIL_FIELDS, CallTypes, CostBreakdown, + LlmProviders, StandardLoggingGuardrailInformation, StandardLoggingMCPToolCall, StandardLoggingModelInformation, @@ -57,6 +60,9 @@ from litellm.types.utils import ( ) from litellm.utils import get_end_user_id_for_cost_tracking +if TYPE_CHECKING: + from litellm.router import Router + def _get_max_string_length_prompt_in_db() -> int: """ @@ -339,12 +345,45 @@ def _sl_attribution_fallback( return standard_logging_payload.get(field) or "" +def _deployment_provider(deployment: DeploymentTypedDict) -> str | None: + litellm_params: Final = LiteLLM_Params.model_validate(deployment["litellm_params"]) + if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default(litellm_params=litellm_params): + return LlmProviders.LITELLM_PROXY.value + declared: Final = declared_authenticating_provider(litellm_params.model, litellm_params.custom_llm_provider) + if declared is not None: + return declared + try: + _, provider, _, _ = litellm.get_llm_provider( + model=litellm_params.model, custom_llm_provider=litellm_params.custom_llm_provider + ) + except litellm.exceptions.BadRequestError: + return None + return provider or None + + +def _model_group_provider(model_group: str, llm_router: "Router | None") -> str | None: + if llm_router is None or not model_group: + return None + providers: Final = frozenset( + provider + for deployment in llm_router.get_model_list(model_name=model_group) or () + if (provider := _deployment_provider(deployment)) is not None + ) + return next(iter(providers)) if len(providers) == 1 else None + + def _looks_like_model_name(model: str) -> bool: candidate: Final = model.removeprefix(MCP_SPEND_LOG_MODEL_PREFIX) return len(candidate) <= MAX_SPEND_LOG_MODEL_NAME_LENGTH and not any(char.isspace() for char in candidate) -def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogsPayload: +def get_logging_payload( + kwargs: dict | None, + response_obj: object, + start_time: datetime, + end_time: datetime, + llm_router: "Router | None" = None, +) -> SpendLogsPayload: if kwargs is None: kwargs = {} @@ -440,15 +479,16 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs hidden_params: Final = standard_logging_payload.get("hidden_params", {}) litellm_overhead_time_ms = hidden_params.get("litellm_overhead_time_ms") - custom_llm_provider: Final = ( + logged_provider: Final = ( kwargs.get("custom_llm_provider") or _sl_attribution_fallback(standard_logging_payload, "custom_llm_provider") or None ) + custom_llm_provider: Final = logged_provider or _model_group_provider(_model_group, llm_router) raw_model: Final = cast(str, kwargs.get("model") or "") resolved_model: Final = ( standard_logging_payload.get("model") if standard_logging_payload is not None else None - ) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) + ) or reconstruct_model_name(raw_model, logged_provider, metadata or {}) failed_with_prompt_shaped_model: Final = ( _get_status_for_spend_log(metadata=metadata) == "failure" and not _model_group diff --git a/litellm/router.py b/litellm/router.py index 5f5522e9fd4..633f060f208 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -109,7 +109,14 @@ from litellm.llms.base_llm.vector_store.transformation import ( RouterVectorStoreEmbeddingExecutor, vector_store_request_metadata, ) +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.llms.openai_like.json_loader import JSONProviderRegistry +from litellm.llms.openai_like.model_info import ( + MODEL_INFO_DISCOVERY_PROVIDERS, + MODEL_INFO_REFRESH_CONCURRENCY, + MODEL_INFO_REFRESH_SECONDS, + get_openai_compatible_model_info, +) from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.router_strategy.least_busy import LeastBusyLoggingHandler from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler @@ -242,6 +249,7 @@ from litellm.types.router import ( Deployment, DeploymentModelListingInfo, DeploymentTypedDict, + DiscoveredDeploymentModelInfo, FallbackAccessCheck, FallbackBudgetCheck, GuardrailTypedDict, @@ -973,6 +981,10 @@ class Router: self.cached_deployment_model_info = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)( self.get_deployment_model_info ) + self._discovered_model_info_cache: InMemoryCache = InMemoryCache( + max_size_in_memory=max(len(model_list or ()), 1), + default_ttl=2 * MODEL_INFO_REFRESH_SECONDS, + ) self._routing_group_rows: tuple[DeploymentTypedDict, ...] | None = None self._init_routing_groups(None) self._provider_unresolved_deployments: tuple[Callable[[], Deployment | None], ...] = () @@ -9492,6 +9504,7 @@ class Router: def set_model_list(self, model_list: list): original_model_list: Final = copy.deepcopy(model_list) + self._discovered_model_info_cache.flush_cache() self.model_list = [] self.model_id_to_deployment_index_map = {} # Reset the index self.model_name_to_deployment_indices = {} # Reset the model_name index @@ -9786,6 +9799,7 @@ class Router: - model_id: str - the id of the deployment that was removed - removal_idx: int - the index where the deployment was removed from model_list """ + self._discovered_model_info_cache.delete_cache(model_id) # Update indices for all models after the removed one for deployment_id, idx in self.model_id_to_deployment_index_map.items(): if idx > removal_idx: @@ -10316,11 +10330,85 @@ class Router: return None return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable + async def arefresh_model_info(self, *, client: AsyncHTTPHandler | None = None) -> None: + """Refresh token limits advertised by configured OpenAI-compatible deployments.""" + deployments: Final = iter(tuple(self.model_list)) + + async def refresh_worker() -> None: + for raw_deployment in deployments: + try: + await self._arefresh_deployment_model_info(raw_deployment, client=client) + except Exception: # noqa: BLE001 # one invalid deployment must not prevent refreshing the others + verbose_router_logger.debug("Could not refresh deployment model info") + + await asyncio.gather(*(refresh_worker() for _ in range(MODEL_INFO_REFRESH_CONCURRENCY))) + self._invalidate_model_group_info_cache() + + async def _arefresh_deployment_model_info( + self, raw_deployment: Mapping[str, object], *, client: AsyncHTTPHandler | None + ) -> None: + deployment: Final = Deployment.model_validate(raw_deployment) + params: Final = LiteLLM_Params.model_validate( + MappingProxyType( + { + **deployment.litellm_params.model_dump(exclude_none=True), + **( + self.get_deployment_credentials_with_provider(deployment.model_info.id or "") + or MappingProxyType({}) + ), + } + ) + ) + model, provider, dynamic_api_key, api_base = litellm.get_llm_provider(model=params.model, litellm_params=params) + if provider not in MODEL_INFO_DISCOVERY_PROVIDERS: + return + if api_base is None or "*" in model or params.get("use_clientside_credentials"): + return + api_key: Final = params.api_key or dynamic_api_key + headers: Final = TypeAdapter(Mapping[str, str]).validate_python( + params.get("extra_headers") or params.get("headers") or MappingProxyType({}) + ) + auth_headers: Final = ( + MappingProxyType({"authorization": f"Bearer {api_key}"}) if api_key else MappingProxyType({}) + ) + limits: Final = await get_openai_compatible_model_info( + model=model, + api_base=api_base, + headers=MappingProxyType( + { + **auth_headers, + **MappingProxyType({key.lower(): value for key, value in headers.items()}), + } + ), + client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI), + cache=self.cache.in_memory_cache, + ) + model_id: Final = deployment.model_info.id + if not limits or model_id is None or self.get_model_info(model_id) is not raw_deployment: + return + self._discovered_model_info_cache.max_size_in_memory = max(len(self.model_list), 1) + self._discovered_model_info_cache.delete_cache(model_id) + self._discovered_model_info_cache.set_cache( + model_id, DiscoveredDeploymentModelInfo(deployment=raw_deployment, limits=limits) + ) + self._invalidate_model_group_info_cache() + + def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: + cached: Final[object] = self._discovered_model_info_cache.get_cache(model_id) + if ( + model_id is not None + and isinstance(cached, DiscoveredDeploymentModelInfo) + and cached.deployment is self.get_model_info(model_id) + ): + configured: Final = TypeAdapter(Mapping[str, object]).validate_python(cached.deployment["model_info"]) + return MappingProxyType({key: value for key, value in cached.limits.items() if configured.get(key) is None}) + return MappingProxyType({}) + def get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None: """ Return what the concrete deployments behind model_name contribute to its /v1/models entry: the cost-map keys for their underlying models, plus the widest - token limits explicitly configured in their model_info. Resolved via O(1) index + configured or discovered token limits. Resolved via O(1) index lookup. Returns None for wildcard-expanded or unknown names, where the listed name is the @@ -10340,7 +10428,21 @@ class Router: return None deployments: Final = tuple(self.model_list[index] for index in indices) - model_infos: Final = tuple(deployment.get("model_info") or MappingProxyType({}) for deployment in deployments) + model_infos: Final = tuple( + MappingProxyType( + { + **self.get_discovered_model_info((deployment.get("model_info") or MappingProxyType({})).get("id")), + **MappingProxyType( + { + k: v + for k, v in (deployment.get("model_info") or MappingProxyType({})).items() + if v is not None + } + ), + } + ) + for deployment in deployments + ) params: Final = tuple(deployment.get("litellm_params") or MappingProxyType({}) for deployment in deployments) # base_model resolution mirrors get_router_model_info: unset or blank means the # deployment's own model name is the cost-map key. @@ -10372,8 +10474,8 @@ class Router: def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]": """ - Return (max_input_tokens, max_output_tokens) explicitly configured in a concrete - deployment's model_info for model_name, via O(1) index lookup. + Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete + deployment of model_name, via O(1) index lookup. Returns (None, None) for wildcard-expanded or unknown names, and treats a malformed configured value as absent rather than failing the caller. @@ -10386,7 +10488,12 @@ class Router: if deployment is None: return (None, None) - model_info: Final = deployment.model_info + model_info: Final = MappingProxyType( + { + **self.get_discovered_model_info(deployment.model_info.id), + **deployment.model_info.model_dump(exclude_none=True), + } + ) return ( coerce_token_limit(model_info.get("max_input_tokens")), coerce_token_limit(model_info.get("max_output_tokens")), @@ -10651,11 +10758,13 @@ class Router: # get_model_info() hands back an lru_cache'd dict, so merge into a copy; unset # values are skipped or Deployment's None pricing defaults would erase the map's - merged_model_info: Final = copy.deepcopy(model_info) - if user_model_info: - for key, value in user_model_info.items(): - if value is not None: - merged_model_info[key] = value + merged_model_info: Final[ModelMapInfo] = { + **copy.deepcopy(model_info), + **self.get_discovered_model_info((deployment.get("model_info") or {}).get("id")), + **MappingProxyType( + {key: value for key, value in (user_model_info or MappingProxyType({})).items() if value is not None} + ), + } return merged_model_info @@ -10702,7 +10811,14 @@ class Router: litellm_model_name_model_info: ModelInfo | None = None try: - custom_model_info = copy.deepcopy(litellm.model_cost.get(model_id)) + custom_model_info = ( + { # mutable-ok: the legacy model-info merge updates this private copy + **copy.deepcopy(litellm.model_cost.get(model_id) or MappingProxyType({})), + **self.get_discovered_model_info(model_id), + } + if model_id in litellm.model_cost + else None + ) except Exception: pass diff --git a/litellm/types/router.py b/litellm/types/router.py index 584d2494db4..592039bd2c0 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -623,6 +623,12 @@ class Deployment(BaseModel): setattr(self, key, value) +@dataclass(frozen=True, slots=True) +class DiscoveredDeploymentModelInfo: + deployment: Mapping[str, object] + limits: Mapping[str, int] + + @dataclass(frozen=True, slots=True) class DeploymentModelListingInfo: """What the deployments behind a model name contribute to its OpenAI-compatible listing entry. diff --git a/tests/test_litellm/litellm_core_utils/test_logging_utils.py b/tests/test_litellm/litellm_core_utils/test_logging_utils.py index f9913f1935d..b446021a7dc 100644 --- a/tests/test_litellm/litellm_core_utils/test_logging_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_logging_utils.py @@ -8,28 +8,28 @@ import pytest from litellm.litellm_core_utils import logging_utils from litellm.litellm_core_utils.logging_utils import ( - _format_base64_size, + format_base64_size, _truncate_base64_in_string, truncate_base64_in_messages, truncate_base64_in_messages_async, ) # --------------------------------------------------------------------------- -# _format_base64_size +# format_base64_size # --------------------------------------------------------------------------- class TestFormatBase64Size: def test_bytes_range(self): - assert _format_base64_size(4) == "3B" + assert format_base64_size(4) == "3B" def test_kb_range(self): # 2000 base64 chars ~ 1500 bytes ~ 1.5KB - assert "KB" in _format_base64_size(2000) + assert "KB" in format_base64_size(2000) def test_mb_range(self): # 2_000_000 base64 chars ~ 1.5MB - result = _format_base64_size(2_000_000) + result = format_base64_size(2_000_000) assert "MB" in result diff --git a/tests/test_litellm/litellm_core_utils/test_safe_json_dumps.py b/tests/test_litellm/litellm_core_utils/test_safe_json_dumps.py index 30385ba758d..1f2664f33cb 100644 --- a/tests/test_litellm/litellm_core_utils/test_safe_json_dumps.py +++ b/tests/test_litellm/litellm_core_utils/test_safe_json_dumps.py @@ -3,7 +3,7 @@ import json import pytest -from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, safe_json_structure, strip_null_bytes def test_primitive_types(): @@ -225,3 +225,14 @@ def test_pydantic_base_model(): assert len(result["healthy_endpoints"]) == 2 assert result["healthy_endpoints"][0]["name"] == "test" assert result["healthy_endpoints"][1] == {"value": 1, "label": "one"} + + +def test_safe_json_structure_keeps_tuples_and_drops_non_string_keys(): + data = {"models": ("a", "b"), "tags": {"y", "x"}, 1: "dropped", "nested": {"deep": ("c",)}} + + structure = safe_json_structure(data, value_transform=lambda key, value: value.upper()) + + assert isinstance(structure, dict) + assert structure == {"models": ("A", "B"), "tags": ["X", "Y"], "nested": {"deep": ("C",)}} + assert type(structure["models"]) is tuple + assert json.loads(safe_dumps(data)) == {"models": ["a", "b"], "tags": ["x", "y"], "nested": {"deep": ["c"]}} diff --git a/tests/test_litellm/llms/openai_like/test_model_info.py b/tests/test_litellm/llms/openai_like/test_model_info.py new file mode 100644 index 00000000000..15a7d9e7fc6 --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_model_info.py @@ -0,0 +1,126 @@ +from collections.abc import Mapping +from typing import Final +from unittest.mock import Mock + +import httpx +import pytest + +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai_like.model_info import ( + MODEL_INFO_REFRESH_SECONDS, + get_openai_compatible_model_info, +) + + +@pytest.mark.parametrize( + ("card", "expected"), + ( + ({"max_model_len": 8192}, {"max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192}), + ( + {"context_length": 4096, "max_output_tokens": 1024}, + {"max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 1024}, + ), + ( + {"max_model_len": 4096, "max_input_tokens": 2048, "max_output_tokens": 8192}, + {"max_tokens": 4096, "max_input_tokens": 2048, "max_output_tokens": 4096}, + ), + ({"max_input_tokens": 2048}, {"max_input_tokens": 2048}), + ({"max_output_tokens": 1024}, {"max_output_tokens": 1024}), + ({"max_model_len": True, "max_output_tokens": -1}, {}), + ({"max_model_len": "8192", "max_input_tokens": 0, "max_output_tokens": 1.5}, {}), + ({}, {}), + ), +) +async def test_discovers_only_valid_advertised_limits(card: Mapping[str, object], expected: Mapping[str, int]) -> None: + def respond(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/tenant/v1/models" + assert request.headers["authorization"] == "Bearer local-key" + return httpx.Response(200, json={"data": [{"id": "org/model", **card}]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + cache: Final = InMemoryCache() + result: Final = await get_openai_compatible_model_info( + model="org/model", + api_base="https://backend.test/tenant/v1/", + headers={"Authorization": "Bearer local-key"}, + client=handler, + cache=cache, + ) + assert result == expected + assert ( + await get_openai_compatible_model_info( + model="missing", + api_base="https://backend.test/tenant/v1/", + headers={"Authorization": "Bearer local-key"}, + client=handler, + cache=cache, + ) + == {} + ) + + +async def test_cache_is_scoped_to_endpoint_and_authentication_and_expires() -> None: + clock: Final = Mock(return_value=0) + responder: Final = Mock( + side_effect=( + httpx.Response( + 200, json={"data": [{"id": "first", "max_model_len": 1024}, {"id": "second", "max_model_len": 2048}]} + ), + httpx.Response(200, json={"data": [{"id": "first", "max_model_len": 4096}]}), + httpx.Response(200, json={"data": [{"id": "first", "max_model_len": 8192}]}), + httpx.Response(200, json={"data": [{"id": "first", "max_model_len": 16384}]}), + ) + ) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client: + handler.client = client + cache: Final = InMemoryCache(clock=clock) + + async def lookup(model: str = "first", host: str = "one.test", key: str = "one") -> Mapping[str, int]: + return await get_openai_compatible_model_info( + model=model, api_base=f"https://{host}", headers={"Authorization": key}, client=handler, cache=cache + ) + + assert (await lookup())["max_input_tokens"] == 1024 + assert (await lookup("second"))["max_input_tokens"] == 2048 + assert responder.call_count == 1 + assert (await lookup(key="two"))["max_input_tokens"] == 4096 + assert (await lookup(host="two.test"))["max_input_tokens"] == 8192 + clock.return_value = MODEL_INFO_REFRESH_SECONDS + 1 + assert (await lookup())["max_input_tokens"] == 16384 + assert responder.call_count == 4 + + +@pytest.mark.parametrize( + "response", + ( + httpx.Response(404), + httpx.Response(401), + httpx.Response(302, headers={"location": "https://elsewhere.test"}), + httpx.Response(200, content=b"not json"), + httpx.Response(200, json={"data": None}), + httpx.ReadTimeout("backend unavailable"), + ), +) +async def test_unavailable_metadata_is_best_effort_and_negative_cached( + response: httpx.Response | Exception, +) -> None: + responder: Final = Mock(side_effect=response if isinstance(response, Exception) else None, return_value=response) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(responder), follow_redirects=True) as client: + handler.client = client + cache: Final = InMemoryCache() + for _ in range(2): + assert ( + await get_openai_compatible_model_info( + model="model", api_base="https://backend.test", headers={}, client=handler, cache=cache + ) + == {} + ) + assert responder.call_count == 1 diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 8f72e1d6248..b3f5a60877d 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -76,6 +76,49 @@ async def test_daily_spend_tracking_with_disabled_spend_logs(): assert call_args["payload"]["custom_llm_provider"] == "openai" +@pytest.mark.asyncio +async def test_update_database_attributes_router_rejected_failure_to_model_group_provider(): + db_writer = DBSpendUpdateWriter() + db_writer._insert_spend_log_to_db = AsyncMock() + db_writer.add_spend_log_transaction_to_daily_user_transaction = AsyncMock() + llm_router: Final = litellm.Router( + model_list=[ + {"model_name": "openai-outage", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-a"}}, + {"model_name": "openai-outage", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-b"}}, + ] + ) + + with ( + patch("litellm.proxy.proxy_server.disable_spend_logs", True), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam + patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam + patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam + patch("litellm.proxy.proxy_server.llm_router", llm_router), # test-quality-ok: get_llm_router reads this proxy_server module global at call time; no injection seam + ): + await db_writer.update_database( + token="test-token", + user_id="test-user", + end_user_id=None, + team_id=None, + org_id=None, + kwargs={ + "model": "openai-outage", + "litellm_params": { + "metadata": {"user_api_key": "test-token", "model_group": "openai-outage", "status": "failure"} + }, + }, + completion_response={}, + start_time=datetime.now(timezone.utc), + end_time=datetime.now(timezone.utc), + response_cost=0.0, + ) + await asyncio.sleep(0) + + payload: Final = db_writer.add_spend_log_transaction_to_daily_user_transaction.call_args[1]["payload"] + assert payload["model_group"] == "openai-outage" + assert payload["custom_llm_provider"] == "openai" + + def _tool_call_response(*names: str) -> object: from types import SimpleNamespace diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py b/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py index 3000a2ea101..a1cf838ab6b 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_model_info.py @@ -9,14 +9,159 @@ Pins (PR2): from __future__ import annotations +import copy +from collections.abc import Callable +from contextlib import AbstractContextManager +from typing import Final from unittest.mock import AsyncMock, MagicMock +import httpx import pytest +from fastapi.testclient import TestClient +import litellm +from litellm.caching.llm_caching_handler import LLMClientCache +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy import proxy_server +from litellm.utils import _invalidate_model_cost_lowercase_map from .conftest import normalize # type: ignore[import-not-found] + +@pytest.mark.parametrize( + ("backend_model", "base_model"), + ( + ("azure/hosted-model", "fallback-model"), + ("openai/org/fallback-model", None), + ("openai/hosted-model", "fallback-model"), + ("openai/fallback-model", "unknown-base-model"), + ), +) +@pytest.mark.parametrize("advertised_limit", (None, 2048)) +async def test_discovery_preserves_model_info_fallbacks( + backend_model: str, base_model: str | None, advertised_limit: int | None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "local", + "litellm_params": { + "model": backend_model, + "api_base": "https://fallback.test/v1", + "api_key": "local-key", + }, + "model_info": {"id": "fallback-deployment", "base_model": base_model, "max_output_tokens": 333}, + } + ] + ) + builtin: Final = { + "litellm_provider": "openai", + "mode": "chat", + "max_input_tokens": 7000, + "max_output_tokens": 2000, + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + } + monkeypatch.setattr( + litellm, + "model_cost", + { + "fallback-model": builtin, + "openai/fallback-model": builtin, + "fallback-deployment": {"litellm_provider": "openai", "mode": "chat"}, + }, + ) + _invalidate_model_cost_lowercase_map() + monkeypatch.setattr(proxy_server, "llm_router", router) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + json={ + "data": [ + { + "id": backend_model.split("/", 1)[1], + "max_model_len": advertised_limit, + } + ] + }, + ) + ) + ) as client: + handler.client = client + await router.arefresh_model_info(client=handler) + deployment: Final = { + **router.model_list[0], + "model_info": {**router.model_list[0]["model_info"], "mode": None}, + } + enriched_models: Final = ( + proxy_server._get_proxy_model_info(copy.deepcopy(deployment)), + proxy_server._enrich_model_info_with_litellm_data(copy.deepcopy(deployment), llm_router=router), + ) + expected_input: Final = ( + advertised_limit + if advertised_limit is not None and backend_model.startswith("openai/") + else builtin["max_input_tokens"] + ) + for enriched in enriched_models: + info: Final = enriched["model_info"] + assert info.get("max_input_tokens") == expected_input + assert info["max_output_tokens"] == 333 + assert info["input_cost_per_token"] == builtin["input_cost_per_token"] + assert info["output_cost_per_token"] == builtin["output_cost_per_token"] + assert info["mode"] is None + _invalidate_model_cost_lowercase_map() + + +async def test_upstream_limits_reach_model_info_routes( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "local", + "litellm_params": { + "model": "hosted_vllm/org/local-model", + "api_base": "https://backend.test/v1", + "api_key": "local-key", + }, + "model_info": {"id": "local-deployment", "max_output_tokens": 512, "max_input_tokens": None}, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + + def respond(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/models" + return httpx.Response(200, json={"data": [{"id": "org/local-model", "max_model_len": 4096}]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as upstream: + handler.client = upstream + litellm.in_memory_llm_clients_cache.set_cache("async_httpx_clientopenai", handler) + await proxy_server.ProxyStartupEvent.refresh_model_info() + with auth_as(): + for path in ("/v1/model/info", "/model/info"): + response: Final = client.get(path) + assert response.status_code == 200, response.text + info: Final = response.json()["data"][0]["model_info"] + assert (info["max_input_tokens"], info["max_output_tokens"]) == (4096, 512) + group_response: Final = client.get("/model_group/info") + assert group_response.status_code == 200, group_response.text + assert group_response.json()["data"][0]["max_input_tokens"] == 4096 + _invalidate_model_cost_lowercase_map() + + # --------------------------------------------------------------------------- # GET /v2/model/info # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 8b105e94d19..1072e970094 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -1,7 +1,7 @@ import asyncio import datetime import json -from collections.abc import Mapping +from collections.abc import Callable, Mapping from datetime import timezone from typing import Any, Final, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -44,6 +44,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( should_store_prompts_and_responses_in_spend_logs, ) from litellm.proxy.utils import hash_token +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( StandardLoggingHiddenParams, StandardLoggingMetadata, @@ -4003,6 +4004,184 @@ def test_get_logging_payload_failed_request_without_standard_logging_payload_lea assert payload["custom_llm_provider"] == "" +def _router_rejected_failure_payload(model_group: str, llm_router: litellm.Router | None) -> SpendLogsPayload: + return get_logging_payload( + kwargs={ + "model": model_group, + "litellm_params": { + "metadata": {"user_api_key": "test-key", "model_group": model_group, "status": "failure"} + }, + }, + response_obj={}, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + llm_router=llm_router, + ) + + +_ProviderResolution = tuple[str, str, str | None, str | None] + + +def _router_init_provider_stub( + model: str, + custom_llm_provider: str | None = None, + api_base: str | None = None, + api_key: str | None = None, + litellm_params: GenericLiteLLMParams | None = None, +) -> _ProviderResolution: + prefix, _, suffix = model.partition("/") + return (suffix or model, custom_llm_provider or (prefix if suffix else "openai"), api_base, api_key) + + +def _oauth_tripwire(resolution_attempts: list[str]) -> Callable[..., _ProviderResolution]: + def _trip( + model: str, + custom_llm_provider: str | None = None, + api_base: str | None = None, + api_key: str | None = None, + litellm_params: GenericLiteLLMParams | None = None, + ) -> _ProviderResolution: + resolution_attempts.append(model) + raise AssertionError("get_llm_provider would run the OAuth device flow") + + return _trip + + +def _openai_and_anthropic_router() -> litellm.Router: + return litellm.Router( + model_list=[ + {"model_name": "openai-group", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-a"}}, + {"model_name": "openai-group", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-b"}}, + {"model_name": "mixed-group", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-a"}}, + { + "model_name": "mixed-group", + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-c"}, + }, + ] + ) + + +@pytest.mark.parametrize( + "model_group,expected_provider", + [("openai-group", "openai"), ("mixed-group", ""), ("not-in-router", "")], +) +def test_get_logging_payload_router_rejected_request_takes_provider_from_model_group( + model_group: str, expected_provider: str +): + payload = _router_rejected_failure_payload(model_group, _openai_and_anthropic_router()) + + assert payload["model_group"] == model_group + assert payload["custom_llm_provider"] == expected_provider + + +def test_get_logging_payload_router_rejected_request_without_router_leaves_provider_empty(): + assert _router_rejected_failure_payload("openai-group", None)["custom_llm_provider"] == "" + + +@pytest.mark.parametrize( + "litellm_params,expected_provider", + [ + ({"model": "github_copilot/gpt-4o"}, "github_copilot"), + ({"model": "gpt-5", "custom_llm_provider": "chatgpt"}, "chatgpt"), + ], +) +def test_get_logging_payload_inferred_provider_never_resolves_declared_authenticating_providers( + monkeypatch: pytest.MonkeyPatch, litellm_params: dict[str, str], expected_provider: str +): + resolution_attempts: list[str] = [] + + monkeypatch.setattr(litellm, "get_llm_provider", _router_init_provider_stub) + llm_router = litellm.Router(model_list=[{"model_name": "oauth-group", "litellm_params": litellm_params}]) + monkeypatch.setattr(litellm, "get_llm_provider", _oauth_tripwire(resolution_attempts)) + + payload = _router_rejected_failure_payload("oauth-group", llm_router) + + assert payload["custom_llm_provider"] == expected_provider + assert resolution_attempts == [] + + +@pytest.mark.parametrize( + "litellm_params", + [ + {"model": "github_copilot/gpt-4o"}, + {"model": "gpt-5", "custom_llm_provider": "chatgpt"}, + {"model": "openai/gpt-4o-mini", "api_key": "sk-a"}, + ], +) +def test_get_logging_payload_inferred_provider_honours_global_litellm_proxy_override( + monkeypatch: pytest.MonkeyPatch, litellm_params: dict[str, str] +): + monkeypatch.setattr(litellm, "get_llm_provider", _router_init_provider_stub) + llm_router = litellm.Router(model_list=[{"model_name": "proxied-group", "litellm_params": litellm_params}]) + monkeypatch.setattr(litellm, "get_llm_provider", _oauth_tripwire([])) + monkeypatch.setattr(litellm, "use_litellm_proxy", True) + + payload = _router_rejected_failure_payload("proxied-group", llm_router) + + assert payload["custom_llm_provider"] == "litellm_proxy" + + +def test_get_logging_payload_router_rejected_request_for_unresolvable_deployment_leaves_provider_empty( + monkeypatch: pytest.MonkeyPatch, +): + with monkeypatch.context() as router_init: + router_init.setattr(litellm, "get_llm_provider", _router_init_provider_stub) + llm_router = litellm.Router( + model_list=[{"model_name": "opaque-group", "litellm_params": {"model": "my-unprefixed-model"}}] + ) + + payload = _router_rejected_failure_payload("opaque-group", llm_router) + + assert payload["model_group"] == "opaque-group" + assert payload["custom_llm_provider"] == "" + + +def test_get_logging_payload_inferred_provider_does_not_rewrite_spend_log_model(): + llm_router = litellm.Router( + model_list=[ + { + "model_name": "bedrock-group", + "litellm_params": { + "model": "bedrock/anthropic.claude-3-5-haiku-20241022-v1:0", + "aws_region_name": "us-east-1", + }, + }, + { + "model_name": "bedrock-group", + "litellm_params": { + "model": "bedrock/anthropic.claude-3-5-haiku-20241022-v1:0", + "aws_region_name": "us-west-2", + }, + }, + ] + ) + + payload = _router_rejected_failure_payload("bedrock-group", llm_router) + + assert payload["custom_llm_provider"] == "bedrock" + assert payload["model"] == "bedrock-group" + + +def test_get_logging_payload_logged_provider_wins_over_model_group_provider(): + payload = get_logging_payload( + kwargs={ + "model": "openai-group", + "litellm_params": {"metadata": {"user_api_key": "test-key", "model_group": "openai-group"}}, + "standard_logging_object": { + **_make_failed_request_standard_logging_payload(), + "model_group": "openai-group", + "custom_llm_provider": "azure", + }, + }, + response_obj={}, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + llm_router=_openai_and_anthropic_router(), + ) + + assert payload["custom_llm_provider"] == "azure" + + class _ModelRouterSpendLogKwargs(TypedDict): model: ReadOnly[str] litellm_params: ReadOnly[dict[str, dict[str, str]]] diff --git a/tests/test_litellm/test_logging.py b/tests/test_litellm/test_logging.py index 57c39280a9f..7cecdaec25d 100644 --- a/tests/test_litellm/test_logging.py +++ b/tests/test_litellm/test_logging.py @@ -1,5 +1,7 @@ import ast import asyncio +import base64 +import dataclasses import json import logging import re @@ -10,12 +12,27 @@ from pathlib import Path from typing import List import pytest +from pydantic import BaseModel, computed_field import litellm from litellm._logging import ( _COLOR_LOG_FORMAT, _MAX_SCRUBBED_ACCESS_ARG, _PLAIN_LOG_FORMAT, + _get_uvicorn_json_log_config, + _initialize_loggers_with_handler, + _parse_json_logs_env, + _plain_log_format, + _stdout_truncation_marker, + _turn_on_json, + format_base64_size, + session_id_var, + set_session_id, + set_trace_id, + trace_id_var, + verbose_logger, + verbose_proxy_logger, + verbose_router_logger, ALL_LOGGERS, AccessLogPathFilter, AccessLogRedactionFilter, @@ -25,22 +42,10 @@ from litellm._logging import ( LevelRoutingStreamHandler, SecretRedactionFilter, StdoutLogTruncationFilter, - _get_uvicorn_json_log_config, - _initialize_loggers_with_handler, - _parse_json_logs_env, - _plain_log_format, - _stdout_truncation_marker, - _turn_on_json, - session_id_var, - set_session_id, - set_trace_id, - trace_id_var, - verbose_logger, - verbose_proxy_logger, - verbose_router_logger, ) from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils import secret_redaction from litellm.types.utils import StandardLoggingPayload @@ -686,10 +691,17 @@ def _make_record(level: int, msg: str, args=(), exc_info=None) -> logging.LogRec ) +def _oversized_text(length: int) -> str: + return ("payload " * (length // 8 + 1))[:length] + + +_OVERSIZED_TEXT = _oversized_text(100_000) + + def test_oversized_info_record_is_truncated(monkeypatch): """An error string echoing a huge request payload must not reach stdout in full.""" monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500") - payload = "p" * 100_000 + payload = _OVERSIZED_TEXT record = _make_record(logging.INFO, "litellm.acompletion(model=%s) Exception %s", ("gpt-4", payload)) assert StdoutLogTruncationFilter().filter(record) is True @@ -697,8 +709,8 @@ def test_oversized_info_record_is_truncated(monkeypatch): message = record.getMessage() assert LITELLM_TRUNCATED_PAYLOAD_FIELD in message assert len(message) <= 500 - assert message.startswith("litellm.acompletion(model=gpt-4) Exception ppp") - assert message.endswith("ppp") + assert message.startswith("litellm.acompletion(model=gpt-4) Exception payload payload") + assert message.endswith("payload ") marker = _extract_marker(message) assert marker is not None @@ -722,7 +734,7 @@ def test_truncated_message_fits_the_configured_cap(monkeypatch): @pytest.mark.parametrize("payload_len", [501, 512, 1000, 9999, 100_000]) def test_truncated_message_never_exceeds_the_cap(monkeypatch, payload_len): monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500") - record = _make_record(logging.ERROR, "%s", ("p" * payload_len,)) + record = _make_record(logging.ERROR, "%s", (_oversized_text(payload_len),)) assert StdoutLogTruncationFilter().filter(record) is True @@ -748,7 +760,7 @@ def test_cap_leaving_no_room_for_the_marker_still_bounds_output(monkeypatch, cap def test_debug_record_is_not_truncated(monkeypatch): """--detailed_debug exists to dump full payloads, so DEBUG records pass through.""" monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500") - payload = "p" * 100_000 + payload = _OVERSIZED_TEXT record = _make_record(logging.DEBUG, "raw request %s", (payload,)) assert StdoutLogTruncationFilter().filter(record) is True @@ -758,7 +770,7 @@ def test_debug_record_is_not_truncated(monkeypatch): def test_truncation_disabled_by_zero_limit(monkeypatch): monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "0") - payload = "p" * 100_000 + payload = _OVERSIZED_TEXT record = _make_record(logging.ERROR, "Exception %s", (payload,)) assert StdoutLogTruncationFilter().filter(record) is True @@ -770,7 +782,7 @@ def test_oversized_traceback_is_truncated(monkeypatch): """verbose_proxy_logger.exception() re-logs the payload inside the traceback too.""" monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500") try: - raise ValueError("payload " + "p" * 100_000) + raise ValueError("payload " + _OVERSIZED_TEXT) except ValueError: exc_info = sys.exc_info() record = _make_record(logging.ERROR, "Exception occured", exc_info=exc_info) @@ -786,7 +798,7 @@ def test_oversized_traceback_is_truncated(monkeypatch): def test_falsy_exc_info_is_not_formatted(monkeypatch): """Callers pass exc_info=False, which logging leaves on the record as a bool.""" monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500") - record = _make_record(logging.WARNING, "skipping malformed endpoint %s", ("p" * 100_000,), exc_info=False) + record = _make_record(logging.WARNING, "skipping malformed endpoint %s", (_OVERSIZED_TEXT,), exc_info=False) assert StdoutLogTruncationFilter().filter(record) is True @@ -799,7 +811,7 @@ def test_secret_filter_keeps_truncated_traceback(monkeypatch): traceback instead of reformatting the full one from exc_info.""" monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500") try: - raise ValueError("sk-1234567890abcdefghij payload " + "p" * 100_000) + raise ValueError("sk-1234567890abcdefghij payload " + _OVERSIZED_TEXT) except ValueError: exc_info = sys.exc_info() record = _make_record(logging.ERROR, "Exception occured", exc_info=exc_info) @@ -825,13 +837,372 @@ def test_oversized_error_is_truncated_end_to_end(monkeypatch, caplog): monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500") with caplog.at_level(logging.INFO, logger="LiteLLM Router"): - verbose_router_logger.info("litellm.acompletion(model=%s) Exception %s", "gpt-4", "p" * 100_000) + verbose_router_logger.info("litellm.acompletion(model=%s) Exception %s", "gpt-4", _OVERSIZED_TEXT) emitted = "".join(record.getMessage() for record in caplog.records) assert LITELLM_TRUNCATED_PAYLOAD_FIELD in emitted assert len(emitted) <= 500 +_PDF_BASE64 = base64.b64encode(bytes(range(256)) * 18).decode() +_IMAGE_BASE64 = base64.b64encode(bytes(range(256)) * 24).decode() +_SHA256_HEX = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" +_LIMIT_SIZED_TOKEN = "t" * 4096 + + +def _base64_run(length: int) -> str: + return (_PDF_BASE64 * (length // len(_PDF_BASE64) + 1))[:length] + + +def test_debug_record_collapses_long_base64_runs(): + """A DEBUG line dumping a document upload keeps its text but not the megabytes of + base64, which cost seconds of event-loop time per line in the secret regex alone.""" + record = _make_record( + logging.DEBUG, + "receiving data: %s", + ( + f"{{'document': 'data:application/pdf;base64,{_PDF_BASE64}', " + f"'base64Source': '{_IMAGE_BASE64}', " + f"'sha256': '{_SHA256_HEX}', 'token': '{_LIMIT_SIZED_TOKEN}'}}", + ), + ) + + assert StdoutLogTruncationFilter().filter(record) is True + + assert record.getMessage() == ( + "receiving data: {'document': 'data:application/pdf;base64,[base64_data truncated: 4.5KB]', " + "'base64Source': '[base64_data truncated: 6.0KB]', " + f"'sha256': '{_SHA256_HEX}', 'token': '{_LIMIT_SIZED_TOKEN}'}}" + ) + + +@pytest.mark.parametrize("run_length,collapses", ((4096, False), (4097, True))) +def test_base64_run_collapses_only_past_the_limit(run_length, collapses): + record = _make_record(logging.DEBUG, "%s", (_base64_run(run_length),)) + + assert StdoutLogTruncationFilter().filter(record) is True + + assert ("[base64_data truncated: " in record.getMessage()) is collapses + + +@pytest.mark.parametrize("limit,collapses", (("0", False), ("100", True))) +def test_base64_collapse_limit_follows_the_env(monkeypatch, limit, collapses): + monkeypatch.setenv("MAX_BASE64_LENGTH_STDOUT_LOG", limit) + record = _make_record(logging.DEBUG, "%s", (_base64_run(200),)) + + assert StdoutLogTruncationFilter().filter(record) is True + + assert ("[base64_data truncated: " in record.getMessage()) is collapses + + +def test_info_record_collapses_base64_before_truncating(monkeypatch): + """The collapse runs at every level ahead of the INFO+ cap, so an error echoing a + document upload comes out as its text around a size placeholder, not a head and tail.""" + monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500") + record = _make_record(logging.ERROR, "Exception: bad document %s (status 400)", (_base64_run(100_000),)) + + assert StdoutLogTruncationFilter().filter(record) is True + + assert record.getMessage() == "Exception: bad document [base64_data truncated: 73.2KB] (status 400)" + + +@pytest.mark.parametrize( + "run", + (_SHA256_HEX * 80, _SHA256_HEX.upper() * 80, "0123456789" * 512, "0f" * 2100), + ids=("hex", "upper_hex", "digits", "two_char_hex_dump"), +) +def test_hex_and_decimal_runs_are_not_mistaken_for_base64(run): + """A long hex dump or numeric id stays in the log line even past the limit, since it + is not a payload and the operator asked for the full debug output.""" + record = _make_record(logging.DEBUG, "checksum %s", (run,)) + + assert StdoutLogTruncationFilter().filter(record) is True + + assert record.getMessage() == f"checksum {run}" + + +@pytest.mark.parametrize( + "payload", + (bytes(6000), b"\x01" * 6000, b"\x55" * 6000, b"\xaa" * 6000), + ids=("zero_filled", "0x01_filled", "0x55_filled", "0xaa_filled"), +) +def test_constant_byte_payloads_still_collapse(payload): + """A zero-filled buffer encodes to one repeated character, and other constant bytes to + a single-case cycle: neither is a digest or an id, so the secret regex never sees them + in full and the event loop is not blocked by a degenerate upload.""" + encoded = base64.b64encode(payload).decode() + record = _make_record(logging.DEBUG, "upload %s", (encoded,)) + + assert StdoutLogTruncationFilter().filter(record) is True + + assert record.getMessage() == f"upload [base64_data truncated: {format_base64_size(len(encoded))}]" + + +def test_debug_traceback_collapses_base64_runs(): + """An exception that echoes a document upload gets the same collapse in its traceback + as the message does, at DEBUG too, so the secret regex never sees the payload in full.""" + try: + raise ValueError(f"bad document: {_base64_run(100_000)}") + except ValueError: + exc_info = sys.exc_info() + record = _make_record(logging.DEBUG, "call failed", exc_info=exc_info) + + assert StdoutLogTruncationFilter().filter(record) is True + + assert record.exc_text is not None + assert "Traceback (most recent call last)" in record.exc_text + assert record.exc_text.endswith("ValueError: bad document: [base64_data truncated: 73.2KB]") + + +def test_base64_collapse_applies_end_to_end(caplog): + """The proxy's own request dump must come out collapsed, not just the filter in isolation.""" + with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"): + verbose_proxy_logger.debug("receiving data: %s", f"{{'document': 'data:application/pdf;base64,{_PDF_BASE64}'}}") + + emitted = "".join(record.getMessage() for record in caplog.records) + assert emitted == "receiving data: {'document': 'data:application/pdf;base64,[base64_data truncated: 4.5KB]'}" + + +class _CountingPattern: + def __init__(self, pattern: "re.Pattern[str]"): + self._pattern = pattern + self.calls = 0 + self.scanned_chars = 0 + + def sub(self, repl: str, string: str, count: int = 0) -> str: + self.calls += 1 + self.scanned_chars += len(string) + return self._pattern.sub(repl, string, count) + + +_REQUEST_DUMP = "{'model': 'gpt-4', 'messages': [{'role': 'user', 'content': 'hello world'}]}" + + +@pytest.mark.parametrize( + "formatter", + (CorrelationPlainFormatter(_PLAIN_LOG_FORMAT), JsonFormatter()), + ids=("plain", "json"), +) +def test_scrubbed_record_is_scanned_for_secrets_once(monkeypatch, formatter): + """Every pass of the secret regex over a multi-megabyte debug line costs seconds of + event-loop time, so a formatter must not rescan what SecretRedactionFilter scrubbed.""" + counting = _CountingPattern(secret_redaction._SECRET_RE) + monkeypatch.setattr(secret_redaction, "_SECRET_RE", counting) + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.DEBUG, "receiving data: %s", (_REQUEST_DUMP,)) + + assert StdoutLogTruncationFilter().filter(record) is True + assert SecretRedactionFilter().filter(record) is True + rendered = formatter.format(record) + + assert _REQUEST_DUMP in rendered + assert "litellm_redacted" not in rendered + assert counting.calls == 1 + assert counting.scanned_chars == len(f"receiving data: {_REQUEST_DUMP}") + + +def test_stamped_record_is_not_scanned_again(monkeypatch): + """JSON mode puts the filter on a third-party logger and again on the root handler its + records propagate to, so the second filter must trust the stamp instead of rescanning.""" + counting = _CountingPattern(secret_redaction._SECRET_RE) + monkeypatch.setattr(secret_redaction, "_SECRET_RE", counting) + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.DEBUG, "receiving data: %s", (_REQUEST_DUMP,)) + + assert SecretRedactionFilter().filter(record) is True + assert SecretRedactionFilter().filter(record) is True + + assert counting.calls == 1 + + +def test_caller_supplied_stamp_never_skips_the_scrub(monkeypatch): + """The stamp is a private sentinel, so a caller passing extra={"litellm_redacted": True} + still gets the full scrub, and only the filter's own stamp lets a later pass skip it.""" + counting = _CountingPattern(secret_redaction._SECRET_RE) + monkeypatch.setattr(secret_redaction, "_SECRET_RE", counting) + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.DEBUG, "api_key=sk-1234567890abcdefghij") + record.litellm_redacted = True + + assert SecretRedactionFilter().filter(record) is True + assert "sk-1234567890abcdefghij" not in record.getMessage() + assert counting.calls == 1 + + assert SecretRedactionFilter().filter(record) is True + assert counting.calls == 1 + + +def test_stack_info_is_scrubbed_before_the_plain_formatter(monkeypatch): + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.INFO, "call failed") + record.stack_info = "Stack (most recent call last):\n api_key=sk-1234567890abcdefghij" + + assert SecretRedactionFilter().filter(record) is True + rendered = CorrelationPlainFormatter(_PLAIN_LOG_FORMAT).format(record) + + assert "sk-1234567890abcdefghij" not in rendered + assert "Stack (most recent call last):" in rendered + + +class _BrokenModel(BaseModel): + name: str + + @computed_field + @property + def snapshot(self) -> str: + raise RuntimeError("snapshot unavailable") + + +@pytest.mark.parametrize( + "extra", + ({1, "a"}, {"nested": {1, "a"}}, _BrokenModel(name="gpt-4o"), {"request": _BrokenModel(name="gpt-4o")}), + ids=("mixed_set", "nested_mixed_set", "raising_model", "nested_raising_model"), +) +def test_unserializable_extra_never_breaks_the_filter(monkeypatch, extra): + """A pydantic computed field that raises escapes model_dump() and str() alike, and a + logging filter that lets it through raises into the caller's own log call.""" + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.WARNING, "request sent") + record.payload = extra + + assert SecretRedactionFilter().filter(record) is True + rendered = json.loads(JsonFormatter().format(record)) + + assert rendered["message"] == "request sent" + assert "payload" in rendered + + +@dataclasses.dataclass(frozen=True, slots=True) +class _RequestExtra: + model: str + attempt: int + api_key: str = dataclasses.field(default="", repr=False) + + +def _nest(value: object, levels: int) -> object: + return value if levels == 0 else _nest([value], levels - 1) + + +@pytest.mark.parametrize( + "extra", + ( + ("gpt-4o", 2), + ["gpt-4o", None, 1.5], + {"models": ("gpt-4o", "gpt-4o-mini"), "attempt": 2}, + {"model": "gpt-4o", "status": "ok"}, + _nest("gpt-4o", 99), + ), + ids=("tuple", "list", "nested_tuple", "dict", "deep_list"), +) +def test_secret_free_extra_keeps_its_original_object(monkeypatch, extra): + """A host application's own handler on a litellm logger reads extras by type, so a + container that carried no secret must reach it untouched, not as its JSON shape.""" + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.WARNING, "request sent") + record.payload = extra + + assert SecretRedactionFilter().filter(record) is True + + assert record.payload is extra + assert "payload" in json.loads(JsonFormatter().format(record)) + + +@pytest.mark.parametrize( + "extra,scrubbed", + ( + (("gpt-4o", "sk-1234567890abcdefghij"), ("gpt-4o", "REDACTED")), + ({"gpt-4o", "sk-1234567890abcdefghij"}, ["REDACTED", "gpt-4o"]), + ({"model": "gpt-4o", "key": "sk-1234567890abcdefghij"}, {"model": "gpt-4o", "key": "REDACTED"}), + ), + ids=("tuple", "set", "dict"), +) +def test_extra_that_carried_a_secret_comes_back_scrubbed(monkeypatch, extra, scrubbed): + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.WARNING, "request sent") + record.payload = extra + + assert SecretRedactionFilter().filter(record) is True + rendered = JsonFormatter().format(record) + + assert record.payload == scrubbed + assert type(record.payload) is type(scrubbed) + assert "sk-1234567890abcdefghij" not in rendered + assert "REDACTED" in rendered + + +class _AmbiguousArray: + def __eq__(self, other: object) -> bool: + raise ValueError("The truth value of an array with more than one element is ambiguous") + + def __repr__(self) -> str: + return "array([1, 2])" + + +@pytest.mark.parametrize( + "extra,scrubbed", + ((_AmbiguousArray(), "array([1, 2])"), ({"weights": _AmbiguousArray()}, {"weights": "array([1, 2])"})), + ids=("top_level", "nested"), +) +def test_extra_whose_equality_raises_still_comes_back_scrubbed(monkeypatch, extra, scrubbed): + """numpy arrays and torch tensors raise when compared for truth, so the keep-or-scrub + decision must fall on the scrubbed copy instead of breaking the caller's log call.""" + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.WARNING, "request sent") + record.payload = extra + + assert SecretRedactionFilter().filter(record) is True + + assert record.payload == scrubbed + assert json.loads(JsonFormatter().format(record))["payload"] == scrubbed + + +@pytest.mark.parametrize( + "extra", + ( + {1: "sk-1234567890abcdefghij"}, + {"model": {1: "sk-1234567890abcdefghij"}}, + _nest("sk-1234567890abcdefghij", 101), + _RequestExtra(model="gpt-4o", attempt=2, api_key="sk-1234567890abcdefghij"), + {"gpt-4o", "sk-1234567890abcdefghij", 1}, + ), + ids=("int_key", "nested_int_key", "deeper_than_safe_dumps", "dataclass_hidden_field", "unsortable_set"), +) +def test_extra_the_filter_cannot_fully_inspect_never_keeps_its_secret(monkeypatch, extra): + """Whatever safe_dumps would skip (non-string keys, anything past its depth limit, + fields a repr hides) must not ride the original object past the redacted stamp.""" + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.WARNING, "request sent") + record.payload = extra + + assert SecretRedactionFilter().filter(record) is True + rendered = JsonFormatter().format(record) + + assert record.payload is not extra + assert "sk-1234567890abcdefghij" not in str(record.payload) + assert "sk-1234567890abcdefghij" not in rendered + + +def test_secret_free_set_comes_back_as_its_json_shape(monkeypatch): + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.WARNING, "request sent") + record.payload = {"gpt-4o", "gpt-4o-mini"} + + assert SecretRedactionFilter().filter(record) is True + + assert record.payload == ["gpt-4o", "gpt-4o-mini"] + assert json.loads(JsonFormatter().format(record))["payload"] == ["gpt-4o", "gpt-4o-mini"] + + +def test_unscrubbed_record_is_still_redacted_by_the_formatter(monkeypatch): + """Records that never met SecretRedactionFilter (uvicorn's, in JSON mode) keep + their formatter-side redaction.""" + monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True) + record = _make_record(logging.INFO, "key sk-1234567890abcdefghij") + + assert "sk-1234567890abcdefghij" not in JsonFormatter().format(record) + assert "sk-1234567890abcdefghij" not in CorrelationPlainFormatter(_PLAIN_LOG_FORMAT).format(record) + + def test_set_session_id_bounds_length(): """set_session_id() must bound length so an oversized caller-supplied value isn't repeated across every log line for the request.""" diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index f097e6f58e5..d73f5efa96b 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -7,18 +7,24 @@ and one has explicit zero-cost pricing in model_info, the other deployment should still use the built-in pricing. """ +import asyncio import copy import logging import os import re -from unittest.mock import patch +from typing import Final +from unittest.mock import Mock, patch +import httpx import pytest - import litellm from litellm import Router +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE from litellm.litellm_core_utils.ptu_pricing import ptu_config_error +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo from litellm.utils import ( _invalidate_model_cost_lowercase_map, @@ -60,6 +66,324 @@ def _restore_model_cost_entries(original_entries): _invalidate_model_cost_lowercase_map() +@pytest.mark.parametrize("initial_count", (1, DEFAULT_MAX_LRU_CACHE_SIZE + 1)) +async def test_discovered_limits_survive_deployment_growth_and_removal( + initial_count: int, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + deployments: Final = tuple( + Deployment( + model_name=f"local-{index}", + litellm_params=LiteLLM_Params( + model="hosted_vllm/local-model", api_base="https://capacity.test/v1", api_key="local-key" + ), + model_info=ModelInfo(id=f"capacity-{index}"), + ) + for index in range(DEFAULT_MAX_LRU_CACHE_SIZE + 2) + ) + router: Final = Router(model_list=[deployment.to_json() for deployment in deployments[:initial_count]]) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}) + ) + ) as client: + handler.client = client + await router.arefresh_model_info(client=handler) + assert all( + router.get_configured_token_limits(deployment.model_name) == (4096, 4096) + for deployment in deployments[:initial_count] + ) + for deployment in deployments[initial_count:]: + router.add_deployment(deployment) + await router._arefresh_deployment_model_info(router.model_list[-1], client=handler) + assert all( + router.get_configured_token_limits(deployment.model_name) == (4096, 4096) for deployment in deployments + ) + for deployment in deployments[-2:]: + router.delete_deployment(deployment.model_info.id or "") + await router._arefresh_deployment_model_info(router.model_list[0], client=handler) + assert all( + router.get_configured_token_limits(deployment.model_name) == (4096, 4096) for deployment in deployments[:-2] + ) + _invalidate_model_cost_lowercase_map() + + +async def test_discovery_discards_metadata_for_a_replaced_deployment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + router: Final = Router(model_list=[{ + "model_name": "local", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://original.test/v1", + "api_key": "local-key", + }, + "model_info": {"id": "replaced-deployment"}, + }]) + + def respond(request: httpx.Request) -> httpx.Response: + if request.url.host == "original.test": + router.upsert_deployment(Deployment( + model_name="local", + litellm_params=LiteLLM_Params( + model="hosted_vllm/local-model", + api_base="https://replacement.test/v1", + api_key="local-key", + ), + model_info=ModelInfo(id="replaced-deployment"), + )) + return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 8192}]}) + assert request.url.host == "replacement.test" + return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 2048}]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + await router._arefresh_deployment_model_info(router.model_list[0], client=handler) + assert router.get_configured_token_limits("local") == (None, None) + await router.arefresh_model_info(client=handler) + assert router.get_configured_token_limits("local") == (2048, 2048) + _invalidate_model_cost_lowercase_map() + + +async def test_discovery_is_isolated_across_routers_and_reused_ids(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + first, second = tuple( + Router(model_list=[{ + "model_name": "local", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": f"https://{host}.test/v1", + "api_key": "local-key", + }, + "model_info": {"id": "shared-discovery-id"}, + }]) + for host in ("first", "second") + ) + + def respond(request: httpx.Request) -> httpx.Response: + if request.url.host == "unavailable.test": + return httpx.Response(503) + limit: Final = 8192 if request.url.host == "first.test" else 2048 + return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": limit}]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + await first.arefresh_model_info(client=handler) + assert second.get_configured_token_limits("local") == (None, None) + await second.arefresh_model_info(client=handler) + assert first.get_discovered_model_info("shared-discovery-id")["max_input_tokens"] == 8192 + assert first.get_configured_token_limits("local") == (8192, 8192) + assert second.get_configured_token_limits("local") == (2048, 2048) + assert litellm.model_cost["shared-discovery-id"].get("max_input_tokens") is None + first.upsert_deployment(Deployment( + model_name="local", + litellm_params=LiteLLM_Params( + model="hosted_vllm/local-model", + api_base="https://unavailable.test/v1", + api_key="local-key", + ), + model_info=ModelInfo(id="shared-discovery-id"), + )) + assert first.get_configured_token_limits("local") == (None, None) + await first.arefresh_model_info(client=handler) + assert first.get_configured_token_limits("local") == (None, None) + assert second.get_configured_token_limits("local") == (2048, 2048) + _invalidate_model_cost_lowercase_map() + + +async def test_discovery_refreshes_other_endpoints_while_one_is_pending(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + second_started: Final = asyncio.Event() + router: Final = Router(model_list=[ + { + "model_name": host, + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": f"https://{host}.test/v1", + "api_key": "local-key", + }, + } + for host in ("first", "second", "third") + ]) + + async def respond(request: httpx.Request) -> httpx.Response: + if request.url.host == "first.test": + await second_started.wait() + if request.url.host == "second.test": + second_started.set() + return httpx.Response(503) + return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 2048}]}) + + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + await asyncio.wait_for(router.arefresh_model_info(client=handler), timeout=2) + assert router.get_configured_token_limits("first") == (2048, 2048) + assert router.get_configured_token_limits("second") == (None, None) + assert router.get_configured_token_limits("third") == (2048, 2048) + _invalidate_model_cost_lowercase_map() + + +async def test_discovered_limits_expire_after_the_last_successful_refresh(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + clock: Final = Mock(return_value=0.0) + router: Final = Router(model_list=[{ + "model_name": "local", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://expiry.test/v1", + "api_key": "local-key", + }, + "model_info": {"id": "expiring-discovery"}, + }]) + router._discovered_model_info_cache = InMemoryCache(clock=clock, default_ttl=2 * MODEL_INFO_REFRESH_SECONDS) + responses: Final = iter(( + httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}), + httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 8192}]}), + httpx.Response(503), + )) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(lambda request: next(responses))) as client: + handler.client = client + await router.arefresh_model_info(client=handler) + clock.return_value = MODEL_INFO_REFRESH_SECONDS + router.cache.in_memory_cache.flush_cache() + await router.arefresh_model_info(client=handler) + clock.return_value = 2 * MODEL_INFO_REFRESH_SECONDS + 1 + router.cache.in_memory_cache.flush_cache() + await router.arefresh_model_info(client=handler) + assert router.get_configured_token_limits("local") == (8192, 8192) + group: Final = router.get_model_group_info("local") + assert group is not None + assert group.max_input_tokens == 8192 + clock.return_value = 3 * MODEL_INFO_REFRESH_SECONDS + 1 + await router.arefresh_model_info(client=handler) + assert router.get_configured_token_limits("local") == (None, None) + expired_group: Final = router.get_model_group_info("local") + assert expired_group is not None + assert expired_group.max_input_tokens is None + _invalidate_model_cost_lowercase_map() + + +@pytest.mark.parametrize("provider", ("hosted_vllm", "openai", "openai_like", "text-completion-openai")) +async def test_discovered_limits_are_isolated_overridable_and_refreshable( + provider: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + upstream_limit: Final = iter((8192, 4096, 16384, 2048)) + + def respond(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/models" + assert request.headers["authorization"] == "Bearer local-key" + return httpx.Response(200, json={"data": [{"id": "org/local-model", "max_model_len": next(upstream_limit)}]}) + + router: Final = Router( + model_list=[ + { + "model_name": "local", + "litellm_params": { + "model": f"{provider}/org/local-model", + "api_base": f"https://{host}.test/v1", + "api_key": "local-key", + }, + "model_info": {"id": host, **overrides}, + } + for host, overrides in (("one", {}), ("two", {"max_output_tokens": 512})) + ], + enable_pre_call_checks=True, + ) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + handler.client = client + await router.arefresh_model_info(client=handler) + first: Final = router.get_router_model_info(id="one", deployment=None, received_model_name="local") + second: Final = router.get_router_model_info(id="two", deployment=None, received_model_name="local") + assert (first["max_input_tokens"], first["max_output_tokens"]) == (8192, 8192) + assert (second["max_input_tokens"], second["max_output_tokens"]) == (4096, 512) + group: Final = router.get_model_group_info("local") + assert group is not None + assert group.max_input_tokens == 8192 + listing: Final = router.get_model_listing_info("local") + assert listing is not None + assert listing.max_input_tokens == 8192 + assert router.get_configured_token_limits("local") == (8192, 8192) + assert router._deployment_max_input_tokens("local", router.model_list[1]) == 4096 + allowed: Final = router._pre_call_checks( + model="local", healthy_deployments=router.model_list, input="prompt", input_token_count=5000 + ) + assert [deployment["model_info"]["id"] for deployment in allowed] == ["one"] + assert router.model_list[0]["model_info"].get("max_input_tokens") is None + assert litellm.model_cost[f"{provider}/org/local-model"].get("max_input_tokens") is None + router.cache.in_memory_cache.flush_cache() + await router.arefresh_model_info(client=handler) + refreshed: Final = router.get_model_group_info("local") + assert refreshed is not None + assert refreshed.max_input_tokens == 16384 + assert ( + router.get_router_model_info(id="two", deployment=None, received_model_name="local")["max_output_tokens"] + == 512 + ) + _invalidate_model_cost_lowercase_map() + + +async def test_discovery_preserves_input_overrides_and_survives_outages(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + responses: Final = iter(( + httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}), + httpx.Response(503), + )) + + def respond(request: httpx.Request) -> httpx.Response: + assert request.url.host == "backend.test" + assert request.headers["authorization"] == "Bearer local-key" + assert request.headers["x-tenant"] == "tenant" + return next(responses) + + router: Final = Router(model_list=[ + { + "model_name": "configured", + "litellm_params": { + "model": "hosted_vllm/local-model", + "api_base": "https://backend.test/v1", + "api_key": "unused-key", + "extra_headers": {"authorization": "Bearer local-key", "X-Tenant": "tenant"}, + }, + "model_info": {"id": "configured", "max_input_tokens": 1024}, + }, + { + "model_name": "byok", + "litellm_params": { + "model": "openai/local-model", + "api_base": "https://caller.test/v1", + "use_clientside_credentials": True, + }, + }, + {"model_name": "default-openai", "litellm_params": {"model": "openai/local-model", "api_key": "unused"}}, + ]) + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + responder: Final = Mock(side_effect=respond) + async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client: + handler.client = client + await router.arefresh_model_info(client=handler) + assert router.get_configured_token_limits("configured") == (1024, 4096) + router.cache.in_memory_cache.flush_cache() + await router.arefresh_model_info(client=handler) + assert router.get_configured_token_limits("configured") == (1024, 4096) + assert router.get_configured_token_limits("byok") == (None, None) + assert next(responses, None) is None + assert responder.call_count == 2 + _invalidate_model_cost_lowercase_map() + + def test_should_not_pollute_shared_key_with_zero_cost_pricing(): """ When deployment A has input_cost_per_token=0 and deployment B has no diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index 9fa748edec1..85933fbf9e8 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -636,11 +636,13 @@ def test_aws_credential_redaction_catches_quoted_values(): {"blob": {"authorization": f"Bearer {SECRET}"}}, {"blob": [f"Bearer {SECRET}"]}, {"blob": ({"nested": {"deep": SECRET}},)}, + {"master_key": "opaque-value-with-no-pattern"}, ), - ids=("set", "dict", "list", "nested"), + ids=("set", "dict", "list", "nested", "key_name"), ) def test_json_formatter_redacts_non_string_extra_values(extra): - """SecretRedactionFilter only scrubs str attrs, so containers must be caught on render.""" + """Container extras and key-named str extras must come out scrubbed, whichever of the + filter and the formatter does the work.""" buf = StringIO() handler = logging.StreamHandler(buf) handler.setFormatter(JsonFormatter())