mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-22 00:31:44 +00:00
Merge remote-tracking branch 'origin/main' into litellm_prompt_injection_async_llm_check
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> # Conflicts: # litellm/proxy/proxy_server.py
This commit is contained in:
yucheng
commit
2cbfd280e9
21 changed files with 1685 additions and 105 deletions
|
|
@ -3,6 +3,7 @@ import contextvars
|
|||
import functools
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from logging import Formatter
|
||||
|
|
@ -13,10 +14,11 @@ import litellm
|
|||
from litellm.constants import (
|
||||
LITELLM_TRUNCATED_PAYLOAD_FIELD,
|
||||
LITELLM_TRUNCATION_STDOUT_SAFEGUARD_NOTE,
|
||||
MAX_BASE64_LENGTH_STDOUT_LOG,
|
||||
MAX_STRING_LENGTH_STDOUT_LOG,
|
||||
)
|
||||
from litellm.litellm_core_utils.env_utils import get_env_int
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.secret_redaction import (
|
||||
redact_internal_details,
|
||||
|
|
@ -77,6 +79,37 @@ def _redact_structured_value(key: str | None, value: str) -> str:
|
|||
return redact_structured_value(key, value)
|
||||
|
||||
|
||||
_REDACTED_RECORD_ATTR: Final = "litellm_redacted"
|
||||
_REDACTED_STAMP: Final = object()
|
||||
_UNREDACTED_SCALAR_TYPES: Final = (bool, int, float, type(None))
|
||||
|
||||
|
||||
def _is_redacted(record: logging.LogRecord) -> bool:
|
||||
return getattr(record, _REDACTED_RECORD_ATTR, None) is _REDACTED_STAMP
|
||||
|
||||
|
||||
def _scrubbing_changed_nothing(scrubbed: object, original: object) -> bool:
|
||||
try:
|
||||
return bool(scrubbed == original)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _plain_text(value: object) -> str:
|
||||
try:
|
||||
return str(value)
|
||||
except Exception:
|
||||
return UNSERIALIZABLE_OBJECT
|
||||
|
||||
|
||||
def _redact_extra_value(key: str, value: object) -> object:
|
||||
try:
|
||||
scrubbed: Final = safe_json_structure(value, value_transform=_redact_structured_value, key=key)
|
||||
except Exception:
|
||||
return _redact_string(_plain_text(value))
|
||||
return value if _scrubbing_changed_nothing(scrubbed, value) else scrubbed
|
||||
|
||||
|
||||
def redact_secrets(value: str) -> str:
|
||||
"""Public API: redact known secret/credential patterns from an arbitrary string.
|
||||
|
||||
|
|
@ -126,7 +159,7 @@ class SecretRedactionFilter(logging.Filter):
|
|||
_formatter = logging.Formatter()
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
if not _ENABLE_SECRET_REDACTION:
|
||||
if not _ENABLE_SECRET_REDACTION or _is_redacted(record):
|
||||
return True
|
||||
|
||||
# Runs before args are cleared, and before the extra-field loop below
|
||||
|
|
@ -149,11 +182,19 @@ class SecretRedactionFilter(logging.Filter):
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
if isinstance(record.stack_info, str):
|
||||
record.stack_info = _redact_string(record.stack_info) # rebind-ok: a Filter scrubs records in place
|
||||
|
||||
# Redact extra fields passed via logger.debug("msg", extra={...})
|
||||
for key, value in list(record.__dict__.items()):
|
||||
if key not in _STANDARD_RECORD_ATTRS and isinstance(value, str):
|
||||
setattr(record, key, _redact_string(value))
|
||||
if key in _STANDARD_RECORD_ATTRS:
|
||||
continue
|
||||
if isinstance(value, str):
|
||||
setattr(record, key, _redact_structured_value(key, value))
|
||||
elif not isinstance(value, _UNREDACTED_SCALAR_TYPES):
|
||||
setattr(record, key, _redact_extra_value(key, value))
|
||||
|
||||
setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP)
|
||||
return True
|
||||
|
||||
|
||||
|
|
@ -277,6 +318,51 @@ def _truncate_for_stdout_log(text: str, limit: int) -> str:
|
|||
return f"{text[:head_chars]}{_stdout_truncation_marker(len(text) - kept_chars)}{text[-tail_chars:]}"
|
||||
|
||||
|
||||
_BYTES_PER_KIB: Final = 1024
|
||||
_BYTES_PER_MIB: Final = 1024 * 1024
|
||||
|
||||
|
||||
def format_base64_size(num_chars: int) -> str:
|
||||
"""Return a human-readable byte-size estimate from a base64 character count."""
|
||||
num_bytes: Final = num_chars * 3 / 4
|
||||
if num_bytes >= _BYTES_PER_MIB:
|
||||
return f"{num_bytes / _BYTES_PER_MIB:.2f}MB"
|
||||
if num_bytes >= _BYTES_PER_KIB:
|
||||
return f"{num_bytes / _BYTES_PER_KIB:.1f}KB"
|
||||
return f"{int(num_bytes)}B"
|
||||
|
||||
|
||||
def _get_max_base64_length_stdout_log() -> int:
|
||||
return get_env_int("MAX_BASE64_LENGTH_STDOUT_LOG", MAX_BASE64_LENGTH_STDOUT_LOG)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=8)
|
||||
def _base64_run_pattern(min_chars: int) -> "re.Pattern[str]":
|
||||
return re.compile(rf"(?<![A-Za-z0-9+/])[A-Za-z0-9+/]{{{min_chars},}}={{0,2}}")
|
||||
|
||||
|
||||
_LOWER_HEX_DIGITS: Final = "0123456789abcdef"
|
||||
_UPPER_HEX_DIGITS: Final = "0123456789ABCDEF"
|
||||
|
||||
|
||||
def _looks_like_base64(run: str) -> bool:
|
||||
unpadded: Final = run.rstrip("=")
|
||||
is_hex_or_decimal: Final = not unpadded.strip(_LOWER_HEX_DIGITS) or not unpadded.strip(_UPPER_HEX_DIGITS)
|
||||
is_one_repeated_char: Final = not unpadded.strip(unpadded[0])
|
||||
return not is_hex_or_decimal or is_one_repeated_char
|
||||
|
||||
|
||||
def _replace_base64_run(match: "re.Match[str]") -> str:
|
||||
run: Final = match.group(0)
|
||||
if not _looks_like_base64(run):
|
||||
return run
|
||||
return f"[base64_data truncated: {format_base64_size(len(run))}]"
|
||||
|
||||
|
||||
def _collapse_base64_runs(text: str, limit: int) -> str:
|
||||
return _base64_run_pattern(limit + 1).sub(_replace_base64_run, text)
|
||||
|
||||
|
||||
class StdoutLogTruncationFilter(logging.Filter):
|
||||
"""Bounds how much of an oversized log line reaches stdout.
|
||||
|
||||
|
|
@ -284,36 +370,42 @@ class StdoutLogTruncationFilter(logging.Filter):
|
|||
request writes hundreds of KB to stdout, repeatedly as the exception propagates from
|
||||
the router to the proxy handler and into its traceback, all inline on the event loop.
|
||||
|
||||
DEBUG records pass through untouched, since dumping full payloads is the point of
|
||||
At every level, in the message and in the traceback alike, a base64 run longer than
|
||||
MAX_BASE64_LENGTH_STDOUT_LOG collapses to a size placeholder first: a multi-megabyte
|
||||
document upload otherwise costs seconds of event-loop time per DEBUG line in the
|
||||
secret regex alone. Hex and decimal runs (digests, numeric ids) are left alone unless
|
||||
they are one repeated character, which is what a zero-filled payload encodes to.
|
||||
The text around a run stays, since dumping payloads is the point of
|
||||
`--detailed_debug`, and logging callbacks (OTEL, Datadog, etc.) don't run through
|
||||
logging filters at all, so they still get the untruncated error.
|
||||
logging filters at all, so they still get the untouched record.
|
||||
"""
|
||||
|
||||
_formatter = logging.Formatter()
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
if record.levelno < logging.INFO:
|
||||
return True
|
||||
|
||||
limit: Final = _get_max_string_length_stdout_log()
|
||||
if limit <= 0:
|
||||
return True
|
||||
|
||||
try:
|
||||
message: Final = record.getMessage()
|
||||
except (TypeError, ValueError):
|
||||
return True
|
||||
|
||||
if len(message) > limit:
|
||||
record.msg = _truncate_for_stdout_log(message, limit) # rebind-ok: the Filter interface mutates the record
|
||||
record.args = None # rebind-ok: args are consumed by the truncated message above
|
||||
base64_limit: Final = _get_max_base64_length_stdout_log()
|
||||
collapsed: Final = _collapse_base64_runs(message, base64_limit) if base64_limit > 0 else message
|
||||
limit: Final = _get_max_string_length_stdout_log() if record.levelno >= logging.INFO else 0
|
||||
bounded: Final = _truncate_for_stdout_log(collapsed, limit) if 0 < limit < len(collapsed) else collapsed
|
||||
if bounded != message:
|
||||
record.msg = bounded # rebind-ok: the Filter interface mutates the record
|
||||
record.args = None # rebind-ok: args are consumed by the rewritten message above
|
||||
|
||||
if isinstance(record.exc_info, tuple):
|
||||
exc_text: Final = record.exc_text or self._formatter.formatException(record.exc_info)
|
||||
if len(exc_text) > limit:
|
||||
record.exc_text = _truncate_for_stdout_log( # rebind-ok: the Filter interface mutates the record
|
||||
exc_text, limit
|
||||
)
|
||||
if not isinstance(record.exc_info, tuple):
|
||||
return True
|
||||
|
||||
exc_text: Final = record.exc_text or self._formatter.formatException(record.exc_info)
|
||||
collapsed_exc: Final = _collapse_base64_runs(exc_text, base64_limit) if base64_limit > 0 else exc_text
|
||||
bounded_exc: Final = (
|
||||
_truncate_for_stdout_log(collapsed_exc, limit) if 0 < limit < len(collapsed_exc) else collapsed_exc
|
||||
)
|
||||
if bounded_exc != exc_text:
|
||||
record.exc_text = bounded_exc # rebind-ok: the Filter interface mutates the record
|
||||
|
||||
return True
|
||||
|
||||
|
|
@ -474,6 +566,7 @@ def _get_standard_record_attrs() -> frozenset:
|
|||
|
||||
|
||||
_STANDARD_RECORD_ATTRS: Final = _get_standard_record_attrs()
|
||||
_NON_EXTRA_RECORD_ATTRS: Final = _STANDARD_RECORD_ATTRS | {_REDACTED_RECORD_ATTR}
|
||||
|
||||
# CorrelationContextFilter is the only legitimate source for these two JSON fields;
|
||||
# see JsonFormatter.format() for why they're excluded from the generic message-content
|
||||
|
|
@ -514,7 +607,7 @@ class JsonFormatter(Formatter):
|
|||
|
||||
# Include extra attributes passed via logger.debug("msg", extra={...})
|
||||
for key, value in record.__dict__.items():
|
||||
if key not in _STANDARD_RECORD_ATTRS and key not in json_record:
|
||||
if key not in _NON_EXTRA_RECORD_ATTRS and key not in json_record:
|
||||
json_record[key] = value
|
||||
|
||||
# trace_id/session_id are reserved: CorrelationContextFilter is the only
|
||||
|
|
@ -538,7 +631,7 @@ class JsonFormatter(Formatter):
|
|||
if record.exc_info:
|
||||
json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info)
|
||||
|
||||
return safe_dumps(json_record, value_transform=_redact_structured_value)
|
||||
return safe_dumps(json_record, value_transform=None if _is_redacted(record) else _redact_structured_value)
|
||||
|
||||
|
||||
class CorrelationPlainFormatter(logging.Formatter):
|
||||
|
|
@ -549,7 +642,8 @@ class CorrelationPlainFormatter(logging.Formatter):
|
|||
"""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
formatted: Final = _redact_string(super().format(record))
|
||||
rendered: Final = super().format(record)
|
||||
formatted: Final = rendered if _is_redacted(record) else _redact_string(rendered)
|
||||
trace_id: Final = getattr(record, "trace_id", None)
|
||||
session_id: Final = getattr(record, "session_id", None)
|
||||
if not trace_id and not session_id:
|
||||
|
|
@ -567,8 +661,8 @@ def _setup_json_exception_handlers(formatter):
|
|||
# Create a handler with JSON formatting for exceptions
|
||||
error_handler: Final = logging.StreamHandler()
|
||||
error_handler.setFormatter(formatter)
|
||||
error_handler.addFilter(_secret_filter)
|
||||
error_handler.addFilter(_stdout_truncation_filter)
|
||||
error_handler.addFilter(_secret_filter)
|
||||
error_handler.addFilter(_correlation_filter)
|
||||
|
||||
# Setup excepthook for uncaught exceptions
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@ REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
|
|||
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
|
||||
|
||||
MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096)
|
||||
MAX_BASE64_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_BASE64_LENGTH_STDOUT_LOG", 4096)
|
||||
|
||||
# When true, adds detailed per-phase timing breakdown headers to responses.
|
||||
# Headers: x-litellm-timing-{pre-processing,llm-api,post-processing,message-copy}-ms
|
||||
|
|
|
|||
|
|
@ -160,7 +160,7 @@ def get_llm_provider(
|
|||
if model is None:
|
||||
raise ValueError("model parameter is required but was None. Please provide a valid model name.")
|
||||
|
||||
if litellm.LiteLLMProxyChatConfig._should_use_litellm_proxy_by_default(
|
||||
if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default(
|
||||
litellm_params=cast(LiteLLM_Params | None, litellm_params)
|
||||
):
|
||||
return litellm.LiteLLMProxyChatConfig.litellm_proxy_get_custom_llm_provider_info(
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from collections.abc import Iterator, Mapping, Sequence
|
|||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._logging import format_base64_size, verbose_logger
|
||||
from litellm.constants import (
|
||||
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS,
|
||||
MAX_BASE64_LENGTH_FOR_LOGGING,
|
||||
|
|
@ -40,9 +40,6 @@ import litellm
|
|||
Helper utils used for logging callbacks
|
||||
"""
|
||||
|
||||
_BYTES_PER_KIB: Final = 1024
|
||||
_BYTES_PER_MIB: Final = 1024 * 1024
|
||||
|
||||
# Regex matching data-URI base64 content: "data:<mime>;base64,<payload>"
|
||||
# Captures: group(1)=mime_type, group(2)=base64_payload
|
||||
_DATA_URI_RE: Final = re.compile(r"data:([^;]+);base64,([A-Za-z0-9+/=]+)")
|
||||
|
|
@ -52,23 +49,13 @@ _DATA_URI_RE: Final = re.compile(r"data:([^;]+);base64,([A-Za-z0-9+/=]+)")
|
|||
_MAX_TRUNCATION_DEPTH: Final = 20
|
||||
|
||||
|
||||
def _format_base64_size(num_chars: int) -> str:
|
||||
"""Return a human-readable byte-size estimate from a base64 character count."""
|
||||
num_bytes: Final = num_chars * 3 / 4
|
||||
if num_bytes >= _BYTES_PER_MIB:
|
||||
return f"{num_bytes / _BYTES_PER_MIB:.2f}MB"
|
||||
if num_bytes >= _BYTES_PER_KIB:
|
||||
return f"{num_bytes / _BYTES_PER_KIB:.1f}KB"
|
||||
return f"{int(num_bytes)}B"
|
||||
|
||||
|
||||
def _base64_data_uri_replacer(match: re.Match) -> str:
|
||||
"""Replace a single base64 data-URI match with a size placeholder if too long."""
|
||||
mime_type: Final = match.group(1)
|
||||
payload: Final = match.group(2)
|
||||
if len(payload) <= MAX_BASE64_LENGTH_FOR_LOGGING:
|
||||
return match.group(0)
|
||||
size_str: Final = _format_base64_size(len(payload))
|
||||
size_str: Final = format_base64_size(len(payload))
|
||||
return f"data:{mime_type};base64,[base64_data truncated: {size_str}]"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,25 +6,29 @@ from pydantic import BaseModel
|
|||
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
|
||||
UNSERIALIZABLE_OBJECT: Final = "Unserializable Object"
|
||||
|
||||
|
||||
def strip_null_bytes(value: str) -> str:
|
||||
"""Strip NUL bytes, which PostgreSQL text/jsonb columns reject (error 22P05)."""
|
||||
return value.replace("\x00", "")
|
||||
|
||||
|
||||
def safe_dumps(
|
||||
data: Any,
|
||||
def safe_json_structure(
|
||||
data: object,
|
||||
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
|
||||
value_transform: Callable[[str | None, str], str] | None = None,
|
||||
) -> str:
|
||||
key: str | None = None,
|
||||
) -> object:
|
||||
"""
|
||||
Recursively serialize data while detecting circular references.
|
||||
Rebuild data out of JSON-native pieces while detecting circular references.
|
||||
If a circular reference is detected then a marker string is returned.
|
||||
NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors.
|
||||
|
||||
value_transform, when given, is applied to every string leaf (and to the
|
||||
str() fallback for non-serializable objects) with the mapping key the leaf
|
||||
was reached under, so callers can rewrite values without touching structure.
|
||||
key is the mapping key data itself was reached under, when the caller has one.
|
||||
"""
|
||||
|
||||
def _transform(key: str | None, value: str) -> str:
|
||||
|
|
@ -75,7 +79,15 @@ def safe_dumps(
|
|||
try:
|
||||
return _transform(key, strip_null_bytes(str(obj)))
|
||||
except Exception:
|
||||
return "Unserializable Object"
|
||||
return UNSERIALIZABLE_OBJECT
|
||||
|
||||
safe_data: Final = _serialize(data, set(), 0)
|
||||
return json.dumps(safe_data, default=str)
|
||||
return _serialize(data, set(), 0, key)
|
||||
|
||||
|
||||
def safe_dumps(
|
||||
data: Any,
|
||||
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
|
||||
value_transform: Callable[[str | None, str], str] | None = None,
|
||||
) -> str:
|
||||
"""Serialize data to JSON text through safe_json_structure."""
|
||||
return json.dumps(safe_json_structure(data, max_depth, value_transform), default=str)
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ class LiteLLMProxyChatConfig(OpenAIGPTConfig):
|
|||
return api_key or get_secret_str("LITELLM_PROXY_API_KEY")
|
||||
|
||||
@staticmethod
|
||||
def _should_use_litellm_proxy_by_default(
|
||||
def should_use_litellm_proxy_by_default(
|
||||
litellm_params: LiteLLM_Params | None = None,
|
||||
):
|
||||
"""
|
||||
|
|
|
|||
92
litellm/llms/openai_like/model_info.py
Normal file
92
litellm/llms/openai_like/model_info.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, BeforeValidator, ConfigDict
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.utils import _add_path_to_api_base # pyright: ignore[reportPrivateUsage] # shared provider URL helper
|
||||
|
||||
MODEL_INFO_REFRESH_SECONDS: Final = 300
|
||||
MODEL_INFO_REFRESH_CONCURRENCY: Final = 8
|
||||
MODEL_INFO_DISCOVERY_PROVIDERS: Final = frozenset({"hosted_vllm", "openai", "text-completion-openai", "openai_like"})
|
||||
_EMPTY_LIMITS: Final[Mapping[str, int]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _positive_limit(value: object) -> int | None:
|
||||
return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None
|
||||
|
||||
|
||||
_TokenLimit: TypeAlias = Annotated[int | None, BeforeValidator(_positive_limit)]
|
||||
|
||||
|
||||
class _ModelCard(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: str
|
||||
max_model_len: _TokenLimit = None
|
||||
context_length: _TokenLimit = None
|
||||
max_input_tokens: _TokenLimit = None
|
||||
max_output_tokens: _TokenLimit = None
|
||||
|
||||
def token_limits(self) -> Mapping[str, int]:
|
||||
context: Final = self.max_model_len or self.context_length
|
||||
input_limit: Final = self.max_input_tokens or context
|
||||
output_limit: Final = self.max_output_tokens or context
|
||||
return MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in (
|
||||
("max_tokens", context),
|
||||
("max_input_tokens", min(input_limit, context) if input_limit and context else input_limit),
|
||||
("max_output_tokens", min(output_limit, context) if output_limit and context else output_limit),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class _ModelList(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
data: tuple[_ModelCard, ...] = ()
|
||||
|
||||
|
||||
async def get_openai_compatible_model_info(
|
||||
*,
|
||||
model: str,
|
||||
api_base: str,
|
||||
headers: Mapping[str, str],
|
||||
client: AsyncHTTPHandler,
|
||||
cache: InMemoryCache,
|
||||
) -> Mapping[str, int]:
|
||||
url: Final = _add_path_to_api_base(api_base, "/v1/models")
|
||||
cache_key: Final = (
|
||||
"upstream_model_info:" + hashlib.sha256(json.dumps((url, sorted(headers.items()))).encode()).hexdigest()
|
||||
)
|
||||
cached: Final[object] = cache.get_cache(cache_key)
|
||||
if isinstance(cached, _ModelList):
|
||||
return next((card.token_limits() for card in cached.data if card.id == model), _EMPTY_LIMITS)
|
||||
|
||||
try:
|
||||
response: Final = await client.get(
|
||||
url=url,
|
||||
headers=dict(headers), # mutable-ok: AsyncHTTPHandler requires a concrete dict
|
||||
timeout=httpx.Timeout(5.0),
|
||||
follow_redirects=False,
|
||||
max_response_bytes=2 * 1024 * 1024,
|
||||
)
|
||||
response.raise_for_status()
|
||||
models: Final = _ModelList.model_validate_json(response.content)
|
||||
except Exception: # noqa: BLE001 # optional upstream metadata must not interrupt proxy refresh
|
||||
verbose_logger.debug("Could not discover upstream model token limits")
|
||||
cache.set_cache(cache_key, _ModelList(), ttl=60)
|
||||
return _EMPTY_LIMITS
|
||||
|
||||
cache.set_cache(cache_key, models, ttl=MODEL_INFO_REFRESH_SECONDS)
|
||||
return next((card.token_limits() for card in models.data if card.id == model), _EMPTY_LIMITS)
|
||||
|
|
@ -257,8 +257,8 @@ class DBSpendUpdateWriter:
|
|||
# Completion object fields
|
||||
kwargs: dict | None,
|
||||
completion_response: object,
|
||||
start_time: datetime | None,
|
||||
end_time: datetime | None,
|
||||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
response_cost: float | None,
|
||||
) -> bool:
|
||||
"""Record the request's spend, answering whether its cost still needs charging.
|
||||
|
|
@ -299,6 +299,7 @@ class DBSpendUpdateWriter:
|
|||
response_obj=completion_response,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
llm_router=get_llm_router(),
|
||||
)
|
||||
payload["spend"] = response_cost or 0.0
|
||||
if isinstance(payload["startTime"], datetime):
|
||||
|
|
|
|||
|
|
@ -305,6 +305,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
|
|||
mask_sensitive_keys,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot
|
||||
from litellm.proxy._types import *
|
||||
|
|
@ -1383,9 +1384,27 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
|
|||
## Initialize shared aiohttp session for connection reuse
|
||||
shared_aiohttp_session = await _initialize_shared_aiohttp_session()
|
||||
|
||||
model_info_scheduler: Final = scheduler if scheduler is not None else AsyncIOScheduler()
|
||||
model_info_scheduler.add_job(
|
||||
ProxyStartupEvent.refresh_model_info,
|
||||
"interval",
|
||||
seconds=MODEL_INFO_REFRESH_SECONDS,
|
||||
id="refresh_model_info",
|
||||
next_run_time=datetime.now(timezone.utc),
|
||||
max_instances=1,
|
||||
replace_existing=True,
|
||||
)
|
||||
if not model_info_scheduler.running:
|
||||
model_info_scheduler.start()
|
||||
|
||||
# End of startup event
|
||||
yield
|
||||
|
||||
if model_info_scheduler.running:
|
||||
model_info_scheduler.remove_job("refresh_model_info")
|
||||
if model_info_scheduler is not scheduler:
|
||||
model_info_scheduler.shutdown(wait=False)
|
||||
|
||||
# Shutdown event - drain in-flight requests before tearing down dependencies
|
||||
# so SIGTERM (rolling update, scale-down, liveness kill) doesn't drop them.
|
||||
GracefulShutdownManager.start_shutdown()
|
||||
|
|
@ -9344,6 +9363,11 @@ class ProxyStartupEvent:
|
|||
if isinstance(callback, _OPTIONAL_PromptInjectionDetection):
|
||||
callback.update_environment(router=llm_router)
|
||||
|
||||
@staticmethod
|
||||
async def refresh_model_info() -> None:
|
||||
if llm_router is not None:
|
||||
await llm_router.arefresh_model_info()
|
||||
|
||||
@staticmethod
|
||||
def _warn_budget_without_db(max_budget: float | None, prisma_client: PrismaClient | None) -> None:
|
||||
if prisma_client is not None or not max_budget or max_budget <= 0:
|
||||
|
|
@ -13600,8 +13624,11 @@ def _enrich_model_info_with_litellm_data(
|
|||
litellm_model_info = litellm.get_model_info(model=litellm_model, custom_llm_provider=split_model[0])
|
||||
except Exception:
|
||||
litellm_model_info = {}
|
||||
for k, v in litellm_model_info.items():
|
||||
if k not in model_info:
|
||||
discovered_model_info: Final = (
|
||||
llm_router.get_discovered_model_info(model_info.get("id")) if llm_router is not None else MappingProxyType({})
|
||||
)
|
||||
for k, v in MappingProxyType({**litellm_model_info, **discovered_model_info}).items():
|
||||
if k not in model_info or (model_info[k] is None and k in discovered_model_info):
|
||||
model_info[k] = v
|
||||
model["model_info"] = model_info
|
||||
# don't return the api key / vertex credentials
|
||||
|
|
@ -15066,8 +15093,11 @@ def _get_proxy_model_info(model: dict) -> dict:
|
|||
litellm_model_info = litellm.get_model_info(model=litellm_model, custom_llm_provider=split_model[0])
|
||||
except Exception:
|
||||
litellm_model_info = {}
|
||||
for k, v in litellm_model_info.items():
|
||||
if k not in model_info:
|
||||
discovered_model_info: Final = (
|
||||
llm_router.get_discovered_model_info(model_info.get("id")) if llm_router is not None else MappingProxyType({})
|
||||
)
|
||||
for k, v in MappingProxyType({**litellm_model_info, **discovered_model_info}).items():
|
||||
if k not in model_info or (model_info[k] is None and k in discovered_model_info):
|
||||
model_info[k] = v
|
||||
model["model_info"] = model_info
|
||||
# don't return the llm credentials
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from collections.abc import Mapping, Sequence
|
|||
from datetime import datetime, timezone
|
||||
from datetime import datetime as dt
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, Protocol, cast, runtime_checkable
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
|
@ -32,6 +32,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
get_litellm_metadata_from_kwargs,
|
||||
reconstruct_model_name,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
coerce_model_access_groups,
|
||||
|
|
@ -43,10 +44,12 @@ from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload, SpendLogsR
|
|||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
|
||||
from litellm.proxy.utils import PrismaClient, hash_token
|
||||
from litellm.types.router import DeploymentTypedDict, LiteLLM_Params
|
||||
from litellm.types.utils import (
|
||||
PROMPT_CARRYING_GUARDRAIL_FIELDS,
|
||||
CallTypes,
|
||||
CostBreakdown,
|
||||
LlmProviders,
|
||||
StandardLoggingGuardrailInformation,
|
||||
StandardLoggingMCPToolCall,
|
||||
StandardLoggingModelInformation,
|
||||
|
|
@ -57,6 +60,9 @@ from litellm.types.utils import (
|
|||
)
|
||||
from litellm.utils import get_end_user_id_for_cost_tracking
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
|
||||
def _get_max_string_length_prompt_in_db() -> int:
|
||||
"""
|
||||
|
|
@ -339,12 +345,45 @@ def _sl_attribution_fallback(
|
|||
return standard_logging_payload.get(field) or ""
|
||||
|
||||
|
||||
def _deployment_provider(deployment: DeploymentTypedDict) -> str | None:
|
||||
litellm_params: Final = LiteLLM_Params.model_validate(deployment["litellm_params"])
|
||||
if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default(litellm_params=litellm_params):
|
||||
return LlmProviders.LITELLM_PROXY.value
|
||||
declared: Final = declared_authenticating_provider(litellm_params.model, litellm_params.custom_llm_provider)
|
||||
if declared is not None:
|
||||
return declared
|
||||
try:
|
||||
_, provider, _, _ = litellm.get_llm_provider(
|
||||
model=litellm_params.model, custom_llm_provider=litellm_params.custom_llm_provider
|
||||
)
|
||||
except litellm.exceptions.BadRequestError:
|
||||
return None
|
||||
return provider or None
|
||||
|
||||
|
||||
def _model_group_provider(model_group: str, llm_router: "Router | None") -> str | None:
|
||||
if llm_router is None or not model_group:
|
||||
return None
|
||||
providers: Final = frozenset(
|
||||
provider
|
||||
for deployment in llm_router.get_model_list(model_name=model_group) or ()
|
||||
if (provider := _deployment_provider(deployment)) is not None
|
||||
)
|
||||
return next(iter(providers)) if len(providers) == 1 else None
|
||||
|
||||
|
||||
def _looks_like_model_name(model: str) -> bool:
|
||||
candidate: Final = model.removeprefix(MCP_SPEND_LOG_MODEL_PREFIX)
|
||||
return len(candidate) <= MAX_SPEND_LOG_MODEL_NAME_LENGTH and not any(char.isspace() for char in candidate)
|
||||
|
||||
|
||||
def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogsPayload:
|
||||
def get_logging_payload(
|
||||
kwargs: dict | None,
|
||||
response_obj: object,
|
||||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
llm_router: "Router | None" = None,
|
||||
) -> SpendLogsPayload:
|
||||
if kwargs is None:
|
||||
kwargs = {}
|
||||
|
||||
|
|
@ -440,15 +479,16 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
|
|||
hidden_params: Final = standard_logging_payload.get("hidden_params", {})
|
||||
litellm_overhead_time_ms = hidden_params.get("litellm_overhead_time_ms")
|
||||
|
||||
custom_llm_provider: Final = (
|
||||
logged_provider: Final = (
|
||||
kwargs.get("custom_llm_provider")
|
||||
or _sl_attribution_fallback(standard_logging_payload, "custom_llm_provider")
|
||||
or None
|
||||
)
|
||||
custom_llm_provider: Final = logged_provider or _model_group_provider(_model_group, llm_router)
|
||||
raw_model: Final = cast(str, kwargs.get("model") or "")
|
||||
resolved_model: Final = (
|
||||
standard_logging_payload.get("model") if standard_logging_payload is not None else None
|
||||
) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {})
|
||||
) or reconstruct_model_name(raw_model, logged_provider, metadata or {})
|
||||
failed_with_prompt_shaped_model: Final = (
|
||||
_get_status_for_spend_log(metadata=metadata) == "failure"
|
||||
and not _model_group
|
||||
|
|
|
|||
|
|
@ -109,7 +109,14 @@ from litellm.llms.base_llm.vector_store.transformation import (
|
|||
RouterVectorStoreEmbeddingExecutor,
|
||||
vector_store_request_metadata,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
from litellm.llms.openai_like.model_info import (
|
||||
MODEL_INFO_DISCOVERY_PROVIDERS,
|
||||
MODEL_INFO_REFRESH_CONCURRENCY,
|
||||
MODEL_INFO_REFRESH_SECONDS,
|
||||
get_openai_compatible_model_info,
|
||||
)
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
|
||||
from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
|
||||
|
|
@ -242,6 +249,7 @@ from litellm.types.router import (
|
|||
Deployment,
|
||||
DeploymentModelListingInfo,
|
||||
DeploymentTypedDict,
|
||||
DiscoveredDeploymentModelInfo,
|
||||
FallbackAccessCheck,
|
||||
FallbackBudgetCheck,
|
||||
GuardrailTypedDict,
|
||||
|
|
@ -973,6 +981,10 @@ class Router:
|
|||
self.cached_deployment_model_info = lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)(
|
||||
self.get_deployment_model_info
|
||||
)
|
||||
self._discovered_model_info_cache: InMemoryCache = InMemoryCache(
|
||||
max_size_in_memory=max(len(model_list or ()), 1),
|
||||
default_ttl=2 * MODEL_INFO_REFRESH_SECONDS,
|
||||
)
|
||||
self._routing_group_rows: tuple[DeploymentTypedDict, ...] | None = None
|
||||
self._init_routing_groups(None)
|
||||
self._provider_unresolved_deployments: tuple[Callable[[], Deployment | None], ...] = ()
|
||||
|
|
@ -9492,6 +9504,7 @@ class Router:
|
|||
|
||||
def set_model_list(self, model_list: list):
|
||||
original_model_list: Final = copy.deepcopy(model_list)
|
||||
self._discovered_model_info_cache.flush_cache()
|
||||
self.model_list = []
|
||||
self.model_id_to_deployment_index_map = {} # Reset the index
|
||||
self.model_name_to_deployment_indices = {} # Reset the model_name index
|
||||
|
|
@ -9786,6 +9799,7 @@ class Router:
|
|||
- model_id: str - the id of the deployment that was removed
|
||||
- removal_idx: int - the index where the deployment was removed from model_list
|
||||
"""
|
||||
self._discovered_model_info_cache.delete_cache(model_id)
|
||||
# Update indices for all models after the removed one
|
||||
for deployment_id, idx in self.model_id_to_deployment_index_map.items():
|
||||
if idx > removal_idx:
|
||||
|
|
@ -10316,11 +10330,85 @@ class Router:
|
|||
return None
|
||||
return Deployment(**first_usable) if isinstance(first_usable, dict) else first_usable
|
||||
|
||||
async def arefresh_model_info(self, *, client: AsyncHTTPHandler | None = None) -> None:
|
||||
"""Refresh token limits advertised by configured OpenAI-compatible deployments."""
|
||||
deployments: Final = iter(tuple(self.model_list))
|
||||
|
||||
async def refresh_worker() -> None:
|
||||
for raw_deployment in deployments:
|
||||
try:
|
||||
await self._arefresh_deployment_model_info(raw_deployment, client=client)
|
||||
except Exception: # noqa: BLE001 # one invalid deployment must not prevent refreshing the others
|
||||
verbose_router_logger.debug("Could not refresh deployment model info")
|
||||
|
||||
await asyncio.gather(*(refresh_worker() for _ in range(MODEL_INFO_REFRESH_CONCURRENCY)))
|
||||
self._invalidate_model_group_info_cache()
|
||||
|
||||
async def _arefresh_deployment_model_info(
|
||||
self, raw_deployment: Mapping[str, object], *, client: AsyncHTTPHandler | None
|
||||
) -> None:
|
||||
deployment: Final = Deployment.model_validate(raw_deployment)
|
||||
params: Final = LiteLLM_Params.model_validate(
|
||||
MappingProxyType(
|
||||
{
|
||||
**deployment.litellm_params.model_dump(exclude_none=True),
|
||||
**(
|
||||
self.get_deployment_credentials_with_provider(deployment.model_info.id or "")
|
||||
or MappingProxyType({})
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
model, provider, dynamic_api_key, api_base = litellm.get_llm_provider(model=params.model, litellm_params=params)
|
||||
if provider not in MODEL_INFO_DISCOVERY_PROVIDERS:
|
||||
return
|
||||
if api_base is None or "*" in model or params.get("use_clientside_credentials"):
|
||||
return
|
||||
api_key: Final = params.api_key or dynamic_api_key
|
||||
headers: Final = TypeAdapter(Mapping[str, str]).validate_python(
|
||||
params.get("extra_headers") or params.get("headers") or MappingProxyType({})
|
||||
)
|
||||
auth_headers: Final = (
|
||||
MappingProxyType({"authorization": f"Bearer {api_key}"}) if api_key else MappingProxyType({})
|
||||
)
|
||||
limits: Final = await get_openai_compatible_model_info(
|
||||
model=model,
|
||||
api_base=api_base,
|
||||
headers=MappingProxyType(
|
||||
{
|
||||
**auth_headers,
|
||||
**MappingProxyType({key.lower(): value for key, value in headers.items()}),
|
||||
}
|
||||
),
|
||||
client=client or get_async_httpx_client(llm_provider=LlmProviders.OPENAI),
|
||||
cache=self.cache.in_memory_cache,
|
||||
)
|
||||
model_id: Final = deployment.model_info.id
|
||||
if not limits or model_id is None or self.get_model_info(model_id) is not raw_deployment:
|
||||
return
|
||||
self._discovered_model_info_cache.max_size_in_memory = max(len(self.model_list), 1)
|
||||
self._discovered_model_info_cache.delete_cache(model_id)
|
||||
self._discovered_model_info_cache.set_cache(
|
||||
model_id, DiscoveredDeploymentModelInfo(deployment=raw_deployment, limits=limits)
|
||||
)
|
||||
self._invalidate_model_group_info_cache()
|
||||
|
||||
def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]:
|
||||
cached: Final[object] = self._discovered_model_info_cache.get_cache(model_id)
|
||||
if (
|
||||
model_id is not None
|
||||
and isinstance(cached, DiscoveredDeploymentModelInfo)
|
||||
and cached.deployment is self.get_model_info(model_id)
|
||||
):
|
||||
configured: Final = TypeAdapter(Mapping[str, object]).validate_python(cached.deployment["model_info"])
|
||||
return MappingProxyType({key: value for key, value in cached.limits.items() if configured.get(key) is None})
|
||||
return MappingProxyType({})
|
||||
|
||||
def get_model_listing_info(self, model_name: str) -> DeploymentModelListingInfo | None:
|
||||
"""
|
||||
Return what the concrete deployments behind model_name contribute to its
|
||||
/v1/models entry: the cost-map keys for their underlying models, plus the widest
|
||||
token limits explicitly configured in their model_info. Resolved via O(1) index
|
||||
configured or discovered token limits. Resolved via O(1) index
|
||||
lookup.
|
||||
|
||||
Returns None for wildcard-expanded or unknown names, where the listed name is the
|
||||
|
|
@ -10340,7 +10428,21 @@ class Router:
|
|||
return None
|
||||
|
||||
deployments: Final = tuple(self.model_list[index] for index in indices)
|
||||
model_infos: Final = tuple(deployment.get("model_info") or MappingProxyType({}) for deployment in deployments)
|
||||
model_infos: Final = tuple(
|
||||
MappingProxyType(
|
||||
{
|
||||
**self.get_discovered_model_info((deployment.get("model_info") or MappingProxyType({})).get("id")),
|
||||
**MappingProxyType(
|
||||
{
|
||||
k: v
|
||||
for k, v in (deployment.get("model_info") or MappingProxyType({})).items()
|
||||
if v is not None
|
||||
}
|
||||
),
|
||||
}
|
||||
)
|
||||
for deployment in deployments
|
||||
)
|
||||
params: Final = tuple(deployment.get("litellm_params") or MappingProxyType({}) for deployment in deployments)
|
||||
# base_model resolution mirrors get_router_model_info: unset or blank means the
|
||||
# deployment's own model name is the cost-map key.
|
||||
|
|
@ -10372,8 +10474,8 @@ class Router:
|
|||
|
||||
def get_configured_token_limits(self, model_name: str) -> "tuple[int | None, int | None]":
|
||||
"""
|
||||
Return (max_input_tokens, max_output_tokens) explicitly configured in a concrete
|
||||
deployment's model_info for model_name, via O(1) index lookup.
|
||||
Return (max_input_tokens, max_output_tokens) configured or discovered for a concrete
|
||||
deployment of model_name, via O(1) index lookup.
|
||||
|
||||
Returns (None, None) for wildcard-expanded or unknown names, and treats a
|
||||
malformed configured value as absent rather than failing the caller.
|
||||
|
|
@ -10386,7 +10488,12 @@ class Router:
|
|||
if deployment is None:
|
||||
return (None, None)
|
||||
|
||||
model_info: Final = deployment.model_info
|
||||
model_info: Final = MappingProxyType(
|
||||
{
|
||||
**self.get_discovered_model_info(deployment.model_info.id),
|
||||
**deployment.model_info.model_dump(exclude_none=True),
|
||||
}
|
||||
)
|
||||
return (
|
||||
coerce_token_limit(model_info.get("max_input_tokens")),
|
||||
coerce_token_limit(model_info.get("max_output_tokens")),
|
||||
|
|
@ -10651,11 +10758,13 @@ class Router:
|
|||
|
||||
# get_model_info() hands back an lru_cache'd dict, so merge into a copy; unset
|
||||
# values are skipped or Deployment's None pricing defaults would erase the map's
|
||||
merged_model_info: Final = copy.deepcopy(model_info)
|
||||
if user_model_info:
|
||||
for key, value in user_model_info.items():
|
||||
if value is not None:
|
||||
merged_model_info[key] = value
|
||||
merged_model_info: Final[ModelMapInfo] = {
|
||||
**copy.deepcopy(model_info),
|
||||
**self.get_discovered_model_info((deployment.get("model_info") or {}).get("id")),
|
||||
**MappingProxyType(
|
||||
{key: value for key, value in (user_model_info or MappingProxyType({})).items() if value is not None}
|
||||
),
|
||||
}
|
||||
|
||||
return merged_model_info
|
||||
|
||||
|
|
@ -10702,7 +10811,14 @@ class Router:
|
|||
litellm_model_name_model_info: ModelInfo | None = None
|
||||
|
||||
try:
|
||||
custom_model_info = copy.deepcopy(litellm.model_cost.get(model_id))
|
||||
custom_model_info = (
|
||||
{ # mutable-ok: the legacy model-info merge updates this private copy
|
||||
**copy.deepcopy(litellm.model_cost.get(model_id) or MappingProxyType({})),
|
||||
**self.get_discovered_model_info(model_id),
|
||||
}
|
||||
if model_id in litellm.model_cost
|
||||
else None
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -623,6 +623,12 @@ class Deployment(BaseModel):
|
|||
setattr(self, key, value)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DiscoveredDeploymentModelInfo:
|
||||
deployment: Mapping[str, object]
|
||||
limits: Mapping[str, int]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeploymentModelListingInfo:
|
||||
"""What the deployments behind a model name contribute to its OpenAI-compatible listing entry.
|
||||
|
|
|
|||
|
|
@ -8,28 +8,28 @@ import pytest
|
|||
|
||||
from litellm.litellm_core_utils import logging_utils
|
||||
from litellm.litellm_core_utils.logging_utils import (
|
||||
_format_base64_size,
|
||||
format_base64_size,
|
||||
_truncate_base64_in_string,
|
||||
truncate_base64_in_messages,
|
||||
truncate_base64_in_messages_async,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _format_base64_size
|
||||
# format_base64_size
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFormatBase64Size:
|
||||
def test_bytes_range(self):
|
||||
assert _format_base64_size(4) == "3B"
|
||||
assert format_base64_size(4) == "3B"
|
||||
|
||||
def test_kb_range(self):
|
||||
# 2000 base64 chars ~ 1500 bytes ~ 1.5KB
|
||||
assert "KB" in _format_base64_size(2000)
|
||||
assert "KB" in format_base64_size(2000)
|
||||
|
||||
def test_mb_range(self):
|
||||
# 2_000_000 base64 chars ~ 1.5MB
|
||||
result = _format_base64_size(2_000_000)
|
||||
result = format_base64_size(2_000_000)
|
||||
assert "MB" in result
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import json
|
|||
import pytest
|
||||
|
||||
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, safe_json_structure, strip_null_bytes
|
||||
|
||||
|
||||
def test_primitive_types():
|
||||
|
|
@ -225,3 +225,14 @@ def test_pydantic_base_model():
|
|||
assert len(result["healthy_endpoints"]) == 2
|
||||
assert result["healthy_endpoints"][0]["name"] == "test"
|
||||
assert result["healthy_endpoints"][1] == {"value": 1, "label": "one"}
|
||||
|
||||
|
||||
def test_safe_json_structure_keeps_tuples_and_drops_non_string_keys():
|
||||
data = {"models": ("a", "b"), "tags": {"y", "x"}, 1: "dropped", "nested": {"deep": ("c",)}}
|
||||
|
||||
structure = safe_json_structure(data, value_transform=lambda key, value: value.upper())
|
||||
|
||||
assert isinstance(structure, dict)
|
||||
assert structure == {"models": ("A", "B"), "tags": ["X", "Y"], "nested": {"deep": ("C",)}}
|
||||
assert type(structure["models"]) is tuple
|
||||
assert json.loads(safe_dumps(data)) == {"models": ["a", "b"], "tags": ["x", "y"], "nested": {"deep": ["c"]}}
|
||||
|
|
|
|||
126
tests/test_litellm/llms/openai_like/test_model_info.py
Normal file
126
tests/test_litellm/llms/openai_like/test_model_info.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from unittest.mock import Mock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.openai_like.model_info import (
|
||||
MODEL_INFO_REFRESH_SECONDS,
|
||||
get_openai_compatible_model_info,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("card", "expected"),
|
||||
(
|
||||
({"max_model_len": 8192}, {"max_tokens": 8192, "max_input_tokens": 8192, "max_output_tokens": 8192}),
|
||||
(
|
||||
{"context_length": 4096, "max_output_tokens": 1024},
|
||||
{"max_tokens": 4096, "max_input_tokens": 4096, "max_output_tokens": 1024},
|
||||
),
|
||||
(
|
||||
{"max_model_len": 4096, "max_input_tokens": 2048, "max_output_tokens": 8192},
|
||||
{"max_tokens": 4096, "max_input_tokens": 2048, "max_output_tokens": 4096},
|
||||
),
|
||||
({"max_input_tokens": 2048}, {"max_input_tokens": 2048}),
|
||||
({"max_output_tokens": 1024}, {"max_output_tokens": 1024}),
|
||||
({"max_model_len": True, "max_output_tokens": -1}, {}),
|
||||
({"max_model_len": "8192", "max_input_tokens": 0, "max_output_tokens": 1.5}, {}),
|
||||
({}, {}),
|
||||
),
|
||||
)
|
||||
async def test_discovers_only_valid_advertised_limits(card: Mapping[str, object], expected: Mapping[str, int]) -> None:
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/tenant/v1/models"
|
||||
assert request.headers["authorization"] == "Bearer local-key"
|
||||
return httpx.Response(200, json={"data": [{"id": "org/model", **card}]})
|
||||
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
|
||||
handler.client = client
|
||||
cache: Final = InMemoryCache()
|
||||
result: Final = await get_openai_compatible_model_info(
|
||||
model="org/model",
|
||||
api_base="https://backend.test/tenant/v1/",
|
||||
headers={"Authorization": "Bearer local-key"},
|
||||
client=handler,
|
||||
cache=cache,
|
||||
)
|
||||
assert result == expected
|
||||
assert (
|
||||
await get_openai_compatible_model_info(
|
||||
model="missing",
|
||||
api_base="https://backend.test/tenant/v1/",
|
||||
headers={"Authorization": "Bearer local-key"},
|
||||
client=handler,
|
||||
cache=cache,
|
||||
)
|
||||
== {}
|
||||
)
|
||||
|
||||
|
||||
async def test_cache_is_scoped_to_endpoint_and_authentication_and_expires() -> None:
|
||||
clock: Final = Mock(return_value=0)
|
||||
responder: Final = Mock(
|
||||
side_effect=(
|
||||
httpx.Response(
|
||||
200, json={"data": [{"id": "first", "max_model_len": 1024}, {"id": "second", "max_model_len": 2048}]}
|
||||
),
|
||||
httpx.Response(200, json={"data": [{"id": "first", "max_model_len": 4096}]}),
|
||||
httpx.Response(200, json={"data": [{"id": "first", "max_model_len": 8192}]}),
|
||||
httpx.Response(200, json={"data": [{"id": "first", "max_model_len": 16384}]}),
|
||||
)
|
||||
)
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client:
|
||||
handler.client = client
|
||||
cache: Final = InMemoryCache(clock=clock)
|
||||
|
||||
async def lookup(model: str = "first", host: str = "one.test", key: str = "one") -> Mapping[str, int]:
|
||||
return await get_openai_compatible_model_info(
|
||||
model=model, api_base=f"https://{host}", headers={"Authorization": key}, client=handler, cache=cache
|
||||
)
|
||||
|
||||
assert (await lookup())["max_input_tokens"] == 1024
|
||||
assert (await lookup("second"))["max_input_tokens"] == 2048
|
||||
assert responder.call_count == 1
|
||||
assert (await lookup(key="two"))["max_input_tokens"] == 4096
|
||||
assert (await lookup(host="two.test"))["max_input_tokens"] == 8192
|
||||
clock.return_value = MODEL_INFO_REFRESH_SECONDS + 1
|
||||
assert (await lookup())["max_input_tokens"] == 16384
|
||||
assert responder.call_count == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response",
|
||||
(
|
||||
httpx.Response(404),
|
||||
httpx.Response(401),
|
||||
httpx.Response(302, headers={"location": "https://elsewhere.test"}),
|
||||
httpx.Response(200, content=b"not json"),
|
||||
httpx.Response(200, json={"data": None}),
|
||||
httpx.ReadTimeout("backend unavailable"),
|
||||
),
|
||||
)
|
||||
async def test_unavailable_metadata_is_best_effort_and_negative_cached(
|
||||
response: httpx.Response | Exception,
|
||||
) -> None:
|
||||
responder: Final = Mock(side_effect=response if isinstance(response, Exception) else None, return_value=response)
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(responder), follow_redirects=True) as client:
|
||||
handler.client = client
|
||||
cache: Final = InMemoryCache()
|
||||
for _ in range(2):
|
||||
assert (
|
||||
await get_openai_compatible_model_info(
|
||||
model="model", api_base="https://backend.test", headers={}, client=handler, cache=cache
|
||||
)
|
||||
== {}
|
||||
)
|
||||
assert responder.call_count == 1
|
||||
|
|
@ -76,6 +76,49 @@ async def test_daily_spend_tracking_with_disabled_spend_logs():
|
|||
assert call_args["payload"]["custom_llm_provider"] == "openai"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_database_attributes_router_rejected_failure_to_model_group_provider():
|
||||
db_writer = DBSpendUpdateWriter()
|
||||
db_writer._insert_spend_log_to_db = AsyncMock()
|
||||
db_writer.add_spend_log_transaction_to_daily_user_transaction = AsyncMock()
|
||||
llm_router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "openai-outage", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-a"}},
|
||||
{"model_name": "openai-outage", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-b"}},
|
||||
]
|
||||
)
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.disable_spend_logs", True), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
|
||||
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
|
||||
patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
|
||||
patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
|
||||
patch("litellm.proxy.proxy_server.llm_router", llm_router), # test-quality-ok: get_llm_router reads this proxy_server module global at call time; no injection seam
|
||||
):
|
||||
await db_writer.update_database(
|
||||
token="test-token",
|
||||
user_id="test-user",
|
||||
end_user_id=None,
|
||||
team_id=None,
|
||||
org_id=None,
|
||||
kwargs={
|
||||
"model": "openai-outage",
|
||||
"litellm_params": {
|
||||
"metadata": {"user_api_key": "test-token", "model_group": "openai-outage", "status": "failure"}
|
||||
},
|
||||
},
|
||||
completion_response={},
|
||||
start_time=datetime.now(timezone.utc),
|
||||
end_time=datetime.now(timezone.utc),
|
||||
response_cost=0.0,
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
payload: Final = db_writer.add_spend_log_transaction_to_daily_user_transaction.call_args[1]["payload"]
|
||||
assert payload["model_group"] == "openai-outage"
|
||||
assert payload["custom_llm_provider"] == "openai"
|
||||
|
||||
|
||||
def _tool_call_response(*names: str) -> object:
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
|
|
|||
|
|
@ -9,14 +9,159 @@ Pins (PR2):
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import litellm
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
from .conftest import normalize # type: ignore[import-not-found]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend_model", "base_model"),
|
||||
(
|
||||
("azure/hosted-model", "fallback-model"),
|
||||
("openai/org/fallback-model", None),
|
||||
("openai/hosted-model", "fallback-model"),
|
||||
("openai/fallback-model", "unknown-base-model"),
|
||||
),
|
||||
)
|
||||
@pytest.mark.parametrize("advertised_limit", (None, 2048))
|
||||
async def test_discovery_preserves_model_info_fallbacks(
|
||||
backend_model: str, base_model: str | None, advertised_limit: int | None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "local",
|
||||
"litellm_params": {
|
||||
"model": backend_model,
|
||||
"api_base": "https://fallback.test/v1",
|
||||
"api_key": "local-key",
|
||||
},
|
||||
"model_info": {"id": "fallback-deployment", "base_model": base_model, "max_output_tokens": 333},
|
||||
}
|
||||
]
|
||||
)
|
||||
builtin: Final = {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"max_input_tokens": 7000,
|
||||
"max_output_tokens": 2000,
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"model_cost",
|
||||
{
|
||||
"fallback-model": builtin,
|
||||
"openai/fallback-model": builtin,
|
||||
"fallback-deployment": {"litellm_provider": "openai", "mode": "chat"},
|
||||
},
|
||||
)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"data": [
|
||||
{
|
||||
"id": backend_model.split("/", 1)[1],
|
||||
"max_model_len": advertised_limit,
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
)
|
||||
) as client:
|
||||
handler.client = client
|
||||
await router.arefresh_model_info(client=handler)
|
||||
deployment: Final = {
|
||||
**router.model_list[0],
|
||||
"model_info": {**router.model_list[0]["model_info"], "mode": None},
|
||||
}
|
||||
enriched_models: Final = (
|
||||
proxy_server._get_proxy_model_info(copy.deepcopy(deployment)),
|
||||
proxy_server._enrich_model_info_with_litellm_data(copy.deepcopy(deployment), llm_router=router),
|
||||
)
|
||||
expected_input: Final = (
|
||||
advertised_limit
|
||||
if advertised_limit is not None and backend_model.startswith("openai/")
|
||||
else builtin["max_input_tokens"]
|
||||
)
|
||||
for enriched in enriched_models:
|
||||
info: Final = enriched["model_info"]
|
||||
assert info.get("max_input_tokens") == expected_input
|
||||
assert info["max_output_tokens"] == 333
|
||||
assert info["input_cost_per_token"] == builtin["input_cost_per_token"]
|
||||
assert info["output_cost_per_token"] == builtin["output_cost_per_token"]
|
||||
assert info["mode"] is None
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
async def test_upstream_limits_reach_model_info_routes(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[object]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "local",
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/org/local-model",
|
||||
"api_base": "https://backend.test/v1",
|
||||
"api_key": "local-key",
|
||||
},
|
||||
"model_info": {"id": "local-deployment", "max_output_tokens": 512, "max_input_tokens": None},
|
||||
}
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list())
|
||||
monkeypatch.setattr(proxy_server, "user_model", None)
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/v1/models"
|
||||
return httpx.Response(200, json={"data": [{"id": "org/local-model", "max_model_len": 4096}]})
|
||||
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as upstream:
|
||||
handler.client = upstream
|
||||
litellm.in_memory_llm_clients_cache.set_cache("async_httpx_clientopenai", handler)
|
||||
await proxy_server.ProxyStartupEvent.refresh_model_info()
|
||||
with auth_as():
|
||||
for path in ("/v1/model/info", "/model/info"):
|
||||
response: Final = client.get(path)
|
||||
assert response.status_code == 200, response.text
|
||||
info: Final = response.json()["data"][0]["model_info"]
|
||||
assert (info["max_input_tokens"], info["max_output_tokens"]) == (4096, 512)
|
||||
group_response: Final = client.get("/model_group/info")
|
||||
assert group_response.status_code == 200, group_response.text
|
||||
assert group_response.json()["data"][0]["max_input_tokens"] == 4096
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET /v2/model/info
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import timezone
|
||||
from typing import Any, Final, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
|
@ -44,6 +44,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
|
|||
should_store_prompts_and_responses_in_spend_logs,
|
||||
)
|
||||
from litellm.proxy.utils import hash_token
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import (
|
||||
StandardLoggingHiddenParams,
|
||||
StandardLoggingMetadata,
|
||||
|
|
@ -4003,6 +4004,184 @@ def test_get_logging_payload_failed_request_without_standard_logging_payload_lea
|
|||
assert payload["custom_llm_provider"] == ""
|
||||
|
||||
|
||||
def _router_rejected_failure_payload(model_group: str, llm_router: litellm.Router | None) -> SpendLogsPayload:
|
||||
return get_logging_payload(
|
||||
kwargs={
|
||||
"model": model_group,
|
||||
"litellm_params": {
|
||||
"metadata": {"user_api_key": "test-key", "model_group": model_group, "status": "failure"}
|
||||
},
|
||||
},
|
||||
response_obj={},
|
||||
start_time=datetime.datetime.now(timezone.utc),
|
||||
end_time=datetime.datetime.now(timezone.utc),
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
|
||||
_ProviderResolution = tuple[str, str, str | None, str | None]
|
||||
|
||||
|
||||
def _router_init_provider_stub(
|
||||
model: str,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
api_key: str | None = None,
|
||||
litellm_params: GenericLiteLLMParams | None = None,
|
||||
) -> _ProviderResolution:
|
||||
prefix, _, suffix = model.partition("/")
|
||||
return (suffix or model, custom_llm_provider or (prefix if suffix else "openai"), api_base, api_key)
|
||||
|
||||
|
||||
def _oauth_tripwire(resolution_attempts: list[str]) -> Callable[..., _ProviderResolution]:
|
||||
def _trip(
|
||||
model: str,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
api_key: str | None = None,
|
||||
litellm_params: GenericLiteLLMParams | None = None,
|
||||
) -> _ProviderResolution:
|
||||
resolution_attempts.append(model)
|
||||
raise AssertionError("get_llm_provider would run the OAuth device flow")
|
||||
|
||||
return _trip
|
||||
|
||||
|
||||
def _openai_and_anthropic_router() -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "openai-group", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-a"}},
|
||||
{"model_name": "openai-group", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-b"}},
|
||||
{"model_name": "mixed-group", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-a"}},
|
||||
{
|
||||
"model_name": "mixed-group",
|
||||
"litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-c"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_group,expected_provider",
|
||||
[("openai-group", "openai"), ("mixed-group", ""), ("not-in-router", "")],
|
||||
)
|
||||
def test_get_logging_payload_router_rejected_request_takes_provider_from_model_group(
|
||||
model_group: str, expected_provider: str
|
||||
):
|
||||
payload = _router_rejected_failure_payload(model_group, _openai_and_anthropic_router())
|
||||
|
||||
assert payload["model_group"] == model_group
|
||||
assert payload["custom_llm_provider"] == expected_provider
|
||||
|
||||
|
||||
def test_get_logging_payload_router_rejected_request_without_router_leaves_provider_empty():
|
||||
assert _router_rejected_failure_payload("openai-group", None)["custom_llm_provider"] == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"litellm_params,expected_provider",
|
||||
[
|
||||
({"model": "github_copilot/gpt-4o"}, "github_copilot"),
|
||||
({"model": "gpt-5", "custom_llm_provider": "chatgpt"}, "chatgpt"),
|
||||
],
|
||||
)
|
||||
def test_get_logging_payload_inferred_provider_never_resolves_declared_authenticating_providers(
|
||||
monkeypatch: pytest.MonkeyPatch, litellm_params: dict[str, str], expected_provider: str
|
||||
):
|
||||
resolution_attempts: list[str] = []
|
||||
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _router_init_provider_stub)
|
||||
llm_router = litellm.Router(model_list=[{"model_name": "oauth-group", "litellm_params": litellm_params}])
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _oauth_tripwire(resolution_attempts))
|
||||
|
||||
payload = _router_rejected_failure_payload("oauth-group", llm_router)
|
||||
|
||||
assert payload["custom_llm_provider"] == expected_provider
|
||||
assert resolution_attempts == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"litellm_params",
|
||||
[
|
||||
{"model": "github_copilot/gpt-4o"},
|
||||
{"model": "gpt-5", "custom_llm_provider": "chatgpt"},
|
||||
{"model": "openai/gpt-4o-mini", "api_key": "sk-a"},
|
||||
],
|
||||
)
|
||||
def test_get_logging_payload_inferred_provider_honours_global_litellm_proxy_override(
|
||||
monkeypatch: pytest.MonkeyPatch, litellm_params: dict[str, str]
|
||||
):
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _router_init_provider_stub)
|
||||
llm_router = litellm.Router(model_list=[{"model_name": "proxied-group", "litellm_params": litellm_params}])
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", _oauth_tripwire([]))
|
||||
monkeypatch.setattr(litellm, "use_litellm_proxy", True)
|
||||
|
||||
payload = _router_rejected_failure_payload("proxied-group", llm_router)
|
||||
|
||||
assert payload["custom_llm_provider"] == "litellm_proxy"
|
||||
|
||||
|
||||
def test_get_logging_payload_router_rejected_request_for_unresolvable_deployment_leaves_provider_empty(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
with monkeypatch.context() as router_init:
|
||||
router_init.setattr(litellm, "get_llm_provider", _router_init_provider_stub)
|
||||
llm_router = litellm.Router(
|
||||
model_list=[{"model_name": "opaque-group", "litellm_params": {"model": "my-unprefixed-model"}}]
|
||||
)
|
||||
|
||||
payload = _router_rejected_failure_payload("opaque-group", llm_router)
|
||||
|
||||
assert payload["model_group"] == "opaque-group"
|
||||
assert payload["custom_llm_provider"] == ""
|
||||
|
||||
|
||||
def test_get_logging_payload_inferred_provider_does_not_rewrite_spend_log_model():
|
||||
llm_router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "bedrock-group",
|
||||
"litellm_params": {
|
||||
"model": "bedrock/anthropic.claude-3-5-haiku-20241022-v1:0",
|
||||
"aws_region_name": "us-east-1",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "bedrock-group",
|
||||
"litellm_params": {
|
||||
"model": "bedrock/anthropic.claude-3-5-haiku-20241022-v1:0",
|
||||
"aws_region_name": "us-west-2",
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
payload = _router_rejected_failure_payload("bedrock-group", llm_router)
|
||||
|
||||
assert payload["custom_llm_provider"] == "bedrock"
|
||||
assert payload["model"] == "bedrock-group"
|
||||
|
||||
|
||||
def test_get_logging_payload_logged_provider_wins_over_model_group_provider():
|
||||
payload = get_logging_payload(
|
||||
kwargs={
|
||||
"model": "openai-group",
|
||||
"litellm_params": {"metadata": {"user_api_key": "test-key", "model_group": "openai-group"}},
|
||||
"standard_logging_object": {
|
||||
**_make_failed_request_standard_logging_payload(),
|
||||
"model_group": "openai-group",
|
||||
"custom_llm_provider": "azure",
|
||||
},
|
||||
},
|
||||
response_obj={},
|
||||
start_time=datetime.datetime.now(timezone.utc),
|
||||
end_time=datetime.datetime.now(timezone.utc),
|
||||
llm_router=_openai_and_anthropic_router(),
|
||||
)
|
||||
|
||||
assert payload["custom_llm_provider"] == "azure"
|
||||
|
||||
|
||||
class _ModelRouterSpendLogKwargs(TypedDict):
|
||||
model: ReadOnly[str]
|
||||
litellm_params: ReadOnly[dict[str, dict[str, str]]]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import ast
|
||||
import asyncio
|
||||
import base64
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
|
@ -10,12 +12,27 @@ from pathlib import Path
|
|||
from typing import List
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, computed_field
|
||||
|
||||
import litellm
|
||||
from litellm._logging import (
|
||||
_COLOR_LOG_FORMAT,
|
||||
_MAX_SCRUBBED_ACCESS_ARG,
|
||||
_PLAIN_LOG_FORMAT,
|
||||
_get_uvicorn_json_log_config,
|
||||
_initialize_loggers_with_handler,
|
||||
_parse_json_logs_env,
|
||||
_plain_log_format,
|
||||
_stdout_truncation_marker,
|
||||
_turn_on_json,
|
||||
format_base64_size,
|
||||
session_id_var,
|
||||
set_session_id,
|
||||
set_trace_id,
|
||||
trace_id_var,
|
||||
verbose_logger,
|
||||
verbose_proxy_logger,
|
||||
verbose_router_logger,
|
||||
ALL_LOGGERS,
|
||||
AccessLogPathFilter,
|
||||
AccessLogRedactionFilter,
|
||||
|
|
@ -25,22 +42,10 @@ from litellm._logging import (
|
|||
LevelRoutingStreamHandler,
|
||||
SecretRedactionFilter,
|
||||
StdoutLogTruncationFilter,
|
||||
_get_uvicorn_json_log_config,
|
||||
_initialize_loggers_with_handler,
|
||||
_parse_json_logs_env,
|
||||
_plain_log_format,
|
||||
_stdout_truncation_marker,
|
||||
_turn_on_json,
|
||||
session_id_var,
|
||||
set_session_id,
|
||||
set_trace_id,
|
||||
trace_id_var,
|
||||
verbose_logger,
|
||||
verbose_proxy_logger,
|
||||
verbose_router_logger,
|
||||
)
|
||||
from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils import secret_redaction
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
||||
|
||||
|
|
@ -686,10 +691,17 @@ def _make_record(level: int, msg: str, args=(), exc_info=None) -> logging.LogRec
|
|||
)
|
||||
|
||||
|
||||
def _oversized_text(length: int) -> str:
|
||||
return ("payload " * (length // 8 + 1))[:length]
|
||||
|
||||
|
||||
_OVERSIZED_TEXT = _oversized_text(100_000)
|
||||
|
||||
|
||||
def test_oversized_info_record_is_truncated(monkeypatch):
|
||||
"""An error string echoing a huge request payload must not reach stdout in full."""
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500")
|
||||
payload = "p" * 100_000
|
||||
payload = _OVERSIZED_TEXT
|
||||
record = _make_record(logging.INFO, "litellm.acompletion(model=%s) Exception %s", ("gpt-4", payload))
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
|
|
@ -697,8 +709,8 @@ def test_oversized_info_record_is_truncated(monkeypatch):
|
|||
message = record.getMessage()
|
||||
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in message
|
||||
assert len(message) <= 500
|
||||
assert message.startswith("litellm.acompletion(model=gpt-4) Exception ppp")
|
||||
assert message.endswith("ppp")
|
||||
assert message.startswith("litellm.acompletion(model=gpt-4) Exception payload payload")
|
||||
assert message.endswith("payload ")
|
||||
|
||||
marker = _extract_marker(message)
|
||||
assert marker is not None
|
||||
|
|
@ -722,7 +734,7 @@ def test_truncated_message_fits_the_configured_cap(monkeypatch):
|
|||
@pytest.mark.parametrize("payload_len", [501, 512, 1000, 9999, 100_000])
|
||||
def test_truncated_message_never_exceeds_the_cap(monkeypatch, payload_len):
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500")
|
||||
record = _make_record(logging.ERROR, "%s", ("p" * payload_len,))
|
||||
record = _make_record(logging.ERROR, "%s", (_oversized_text(payload_len),))
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
|
||||
|
|
@ -748,7 +760,7 @@ def test_cap_leaving_no_room_for_the_marker_still_bounds_output(monkeypatch, cap
|
|||
def test_debug_record_is_not_truncated(monkeypatch):
|
||||
"""--detailed_debug exists to dump full payloads, so DEBUG records pass through."""
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500")
|
||||
payload = "p" * 100_000
|
||||
payload = _OVERSIZED_TEXT
|
||||
record = _make_record(logging.DEBUG, "raw request %s", (payload,))
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
|
|
@ -758,7 +770,7 @@ def test_debug_record_is_not_truncated(monkeypatch):
|
|||
|
||||
def test_truncation_disabled_by_zero_limit(monkeypatch):
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "0")
|
||||
payload = "p" * 100_000
|
||||
payload = _OVERSIZED_TEXT
|
||||
record = _make_record(logging.ERROR, "Exception %s", (payload,))
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
|
|
@ -770,7 +782,7 @@ def test_oversized_traceback_is_truncated(monkeypatch):
|
|||
"""verbose_proxy_logger.exception() re-logs the payload inside the traceback too."""
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500")
|
||||
try:
|
||||
raise ValueError("payload " + "p" * 100_000)
|
||||
raise ValueError("payload " + _OVERSIZED_TEXT)
|
||||
except ValueError:
|
||||
exc_info = sys.exc_info()
|
||||
record = _make_record(logging.ERROR, "Exception occured", exc_info=exc_info)
|
||||
|
|
@ -786,7 +798,7 @@ def test_oversized_traceback_is_truncated(monkeypatch):
|
|||
def test_falsy_exc_info_is_not_formatted(monkeypatch):
|
||||
"""Callers pass exc_info=False, which logging leaves on the record as a bool."""
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500")
|
||||
record = _make_record(logging.WARNING, "skipping malformed endpoint %s", ("p" * 100_000,), exc_info=False)
|
||||
record = _make_record(logging.WARNING, "skipping malformed endpoint %s", (_OVERSIZED_TEXT,), exc_info=False)
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
|
||||
|
|
@ -799,7 +811,7 @@ def test_secret_filter_keeps_truncated_traceback(monkeypatch):
|
|||
traceback instead of reformatting the full one from exc_info."""
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500")
|
||||
try:
|
||||
raise ValueError("sk-1234567890abcdefghij payload " + "p" * 100_000)
|
||||
raise ValueError("sk-1234567890abcdefghij payload " + _OVERSIZED_TEXT)
|
||||
except ValueError:
|
||||
exc_info = sys.exc_info()
|
||||
record = _make_record(logging.ERROR, "Exception occured", exc_info=exc_info)
|
||||
|
|
@ -825,13 +837,372 @@ def test_oversized_error_is_truncated_end_to_end(monkeypatch, caplog):
|
|||
monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500")
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="LiteLLM Router"):
|
||||
verbose_router_logger.info("litellm.acompletion(model=%s) Exception %s", "gpt-4", "p" * 100_000)
|
||||
verbose_router_logger.info("litellm.acompletion(model=%s) Exception %s", "gpt-4", _OVERSIZED_TEXT)
|
||||
|
||||
emitted = "".join(record.getMessage() for record in caplog.records)
|
||||
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in emitted
|
||||
assert len(emitted) <= 500
|
||||
|
||||
|
||||
_PDF_BASE64 = base64.b64encode(bytes(range(256)) * 18).decode()
|
||||
_IMAGE_BASE64 = base64.b64encode(bytes(range(256)) * 24).decode()
|
||||
_SHA256_HEX = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
|
||||
_LIMIT_SIZED_TOKEN = "t" * 4096
|
||||
|
||||
|
||||
def _base64_run(length: int) -> str:
|
||||
return (_PDF_BASE64 * (length // len(_PDF_BASE64) + 1))[:length]
|
||||
|
||||
|
||||
def test_debug_record_collapses_long_base64_runs():
|
||||
"""A DEBUG line dumping a document upload keeps its text but not the megabytes of
|
||||
base64, which cost seconds of event-loop time per line in the secret regex alone."""
|
||||
record = _make_record(
|
||||
logging.DEBUG,
|
||||
"receiving data: %s",
|
||||
(
|
||||
f"{{'document': 'data:application/pdf;base64,{_PDF_BASE64}', "
|
||||
f"'base64Source': '{_IMAGE_BASE64}', "
|
||||
f"'sha256': '{_SHA256_HEX}', 'token': '{_LIMIT_SIZED_TOKEN}'}}",
|
||||
),
|
||||
)
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
|
||||
assert record.getMessage() == (
|
||||
"receiving data: {'document': 'data:application/pdf;base64,[base64_data truncated: 4.5KB]', "
|
||||
"'base64Source': '[base64_data truncated: 6.0KB]', "
|
||||
f"'sha256': '{_SHA256_HEX}', 'token': '{_LIMIT_SIZED_TOKEN}'}}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("run_length,collapses", ((4096, False), (4097, True)))
|
||||
def test_base64_run_collapses_only_past_the_limit(run_length, collapses):
|
||||
record = _make_record(logging.DEBUG, "%s", (_base64_run(run_length),))
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
|
||||
assert ("[base64_data truncated: " in record.getMessage()) is collapses
|
||||
|
||||
|
||||
@pytest.mark.parametrize("limit,collapses", (("0", False), ("100", True)))
|
||||
def test_base64_collapse_limit_follows_the_env(monkeypatch, limit, collapses):
|
||||
monkeypatch.setenv("MAX_BASE64_LENGTH_STDOUT_LOG", limit)
|
||||
record = _make_record(logging.DEBUG, "%s", (_base64_run(200),))
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
|
||||
assert ("[base64_data truncated: " in record.getMessage()) is collapses
|
||||
|
||||
|
||||
def test_info_record_collapses_base64_before_truncating(monkeypatch):
|
||||
"""The collapse runs at every level ahead of the INFO+ cap, so an error echoing a
|
||||
document upload comes out as its text around a size placeholder, not a head and tail."""
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500")
|
||||
record = _make_record(logging.ERROR, "Exception: bad document %s (status 400)", (_base64_run(100_000),))
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
|
||||
assert record.getMessage() == "Exception: bad document [base64_data truncated: 73.2KB] (status 400)"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"run",
|
||||
(_SHA256_HEX * 80, _SHA256_HEX.upper() * 80, "0123456789" * 512, "0f" * 2100),
|
||||
ids=("hex", "upper_hex", "digits", "two_char_hex_dump"),
|
||||
)
|
||||
def test_hex_and_decimal_runs_are_not_mistaken_for_base64(run):
|
||||
"""A long hex dump or numeric id stays in the log line even past the limit, since it
|
||||
is not a payload and the operator asked for the full debug output."""
|
||||
record = _make_record(logging.DEBUG, "checksum %s", (run,))
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
|
||||
assert record.getMessage() == f"checksum {run}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
(bytes(6000), b"\x01" * 6000, b"\x55" * 6000, b"\xaa" * 6000),
|
||||
ids=("zero_filled", "0x01_filled", "0x55_filled", "0xaa_filled"),
|
||||
)
|
||||
def test_constant_byte_payloads_still_collapse(payload):
|
||||
"""A zero-filled buffer encodes to one repeated character, and other constant bytes to
|
||||
a single-case cycle: neither is a digest or an id, so the secret regex never sees them
|
||||
in full and the event loop is not blocked by a degenerate upload."""
|
||||
encoded = base64.b64encode(payload).decode()
|
||||
record = _make_record(logging.DEBUG, "upload %s", (encoded,))
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
|
||||
assert record.getMessage() == f"upload [base64_data truncated: {format_base64_size(len(encoded))}]"
|
||||
|
||||
|
||||
def test_debug_traceback_collapses_base64_runs():
|
||||
"""An exception that echoes a document upload gets the same collapse in its traceback
|
||||
as the message does, at DEBUG too, so the secret regex never sees the payload in full."""
|
||||
try:
|
||||
raise ValueError(f"bad document: {_base64_run(100_000)}")
|
||||
except ValueError:
|
||||
exc_info = sys.exc_info()
|
||||
record = _make_record(logging.DEBUG, "call failed", exc_info=exc_info)
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
|
||||
assert record.exc_text is not None
|
||||
assert "Traceback (most recent call last)" in record.exc_text
|
||||
assert record.exc_text.endswith("ValueError: bad document: [base64_data truncated: 73.2KB]")
|
||||
|
||||
|
||||
def test_base64_collapse_applies_end_to_end(caplog):
|
||||
"""The proxy's own request dump must come out collapsed, not just the filter in isolation."""
|
||||
with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
|
||||
verbose_proxy_logger.debug("receiving data: %s", f"{{'document': 'data:application/pdf;base64,{_PDF_BASE64}'}}")
|
||||
|
||||
emitted = "".join(record.getMessage() for record in caplog.records)
|
||||
assert emitted == "receiving data: {'document': 'data:application/pdf;base64,[base64_data truncated: 4.5KB]'}"
|
||||
|
||||
|
||||
class _CountingPattern:
|
||||
def __init__(self, pattern: "re.Pattern[str]"):
|
||||
self._pattern = pattern
|
||||
self.calls = 0
|
||||
self.scanned_chars = 0
|
||||
|
||||
def sub(self, repl: str, string: str, count: int = 0) -> str:
|
||||
self.calls += 1
|
||||
self.scanned_chars += len(string)
|
||||
return self._pattern.sub(repl, string, count)
|
||||
|
||||
|
||||
_REQUEST_DUMP = "{'model': 'gpt-4', 'messages': [{'role': 'user', 'content': 'hello world'}]}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"formatter",
|
||||
(CorrelationPlainFormatter(_PLAIN_LOG_FORMAT), JsonFormatter()),
|
||||
ids=("plain", "json"),
|
||||
)
|
||||
def test_scrubbed_record_is_scanned_for_secrets_once(monkeypatch, formatter):
|
||||
"""Every pass of the secret regex over a multi-megabyte debug line costs seconds of
|
||||
event-loop time, so a formatter must not rescan what SecretRedactionFilter scrubbed."""
|
||||
counting = _CountingPattern(secret_redaction._SECRET_RE)
|
||||
monkeypatch.setattr(secret_redaction, "_SECRET_RE", counting)
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.DEBUG, "receiving data: %s", (_REQUEST_DUMP,))
|
||||
|
||||
assert StdoutLogTruncationFilter().filter(record) is True
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
rendered = formatter.format(record)
|
||||
|
||||
assert _REQUEST_DUMP in rendered
|
||||
assert "litellm_redacted" not in rendered
|
||||
assert counting.calls == 1
|
||||
assert counting.scanned_chars == len(f"receiving data: {_REQUEST_DUMP}")
|
||||
|
||||
|
||||
def test_stamped_record_is_not_scanned_again(monkeypatch):
|
||||
"""JSON mode puts the filter on a third-party logger and again on the root handler its
|
||||
records propagate to, so the second filter must trust the stamp instead of rescanning."""
|
||||
counting = _CountingPattern(secret_redaction._SECRET_RE)
|
||||
monkeypatch.setattr(secret_redaction, "_SECRET_RE", counting)
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.DEBUG, "receiving data: %s", (_REQUEST_DUMP,))
|
||||
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
|
||||
assert counting.calls == 1
|
||||
|
||||
|
||||
def test_caller_supplied_stamp_never_skips_the_scrub(monkeypatch):
|
||||
"""The stamp is a private sentinel, so a caller passing extra={"litellm_redacted": True}
|
||||
still gets the full scrub, and only the filter's own stamp lets a later pass skip it."""
|
||||
counting = _CountingPattern(secret_redaction._SECRET_RE)
|
||||
monkeypatch.setattr(secret_redaction, "_SECRET_RE", counting)
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.DEBUG, "api_key=sk-1234567890abcdefghij")
|
||||
record.litellm_redacted = True
|
||||
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
assert "sk-1234567890abcdefghij" not in record.getMessage()
|
||||
assert counting.calls == 1
|
||||
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
assert counting.calls == 1
|
||||
|
||||
|
||||
def test_stack_info_is_scrubbed_before_the_plain_formatter(monkeypatch):
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.INFO, "call failed")
|
||||
record.stack_info = "Stack (most recent call last):\n api_key=sk-1234567890abcdefghij"
|
||||
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
rendered = CorrelationPlainFormatter(_PLAIN_LOG_FORMAT).format(record)
|
||||
|
||||
assert "sk-1234567890abcdefghij" not in rendered
|
||||
assert "Stack (most recent call last):" in rendered
|
||||
|
||||
|
||||
class _BrokenModel(BaseModel):
|
||||
name: str
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def snapshot(self) -> str:
|
||||
raise RuntimeError("snapshot unavailable")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra",
|
||||
({1, "a"}, {"nested": {1, "a"}}, _BrokenModel(name="gpt-4o"), {"request": _BrokenModel(name="gpt-4o")}),
|
||||
ids=("mixed_set", "nested_mixed_set", "raising_model", "nested_raising_model"),
|
||||
)
|
||||
def test_unserializable_extra_never_breaks_the_filter(monkeypatch, extra):
|
||||
"""A pydantic computed field that raises escapes model_dump() and str() alike, and a
|
||||
logging filter that lets it through raises into the caller's own log call."""
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.WARNING, "request sent")
|
||||
record.payload = extra
|
||||
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
rendered = json.loads(JsonFormatter().format(record))
|
||||
|
||||
assert rendered["message"] == "request sent"
|
||||
assert "payload" in rendered
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class _RequestExtra:
|
||||
model: str
|
||||
attempt: int
|
||||
api_key: str = dataclasses.field(default="", repr=False)
|
||||
|
||||
|
||||
def _nest(value: object, levels: int) -> object:
|
||||
return value if levels == 0 else _nest([value], levels - 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra",
|
||||
(
|
||||
("gpt-4o", 2),
|
||||
["gpt-4o", None, 1.5],
|
||||
{"models": ("gpt-4o", "gpt-4o-mini"), "attempt": 2},
|
||||
{"model": "gpt-4o", "status": "ok"},
|
||||
_nest("gpt-4o", 99),
|
||||
),
|
||||
ids=("tuple", "list", "nested_tuple", "dict", "deep_list"),
|
||||
)
|
||||
def test_secret_free_extra_keeps_its_original_object(monkeypatch, extra):
|
||||
"""A host application's own handler on a litellm logger reads extras by type, so a
|
||||
container that carried no secret must reach it untouched, not as its JSON shape."""
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.WARNING, "request sent")
|
||||
record.payload = extra
|
||||
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
|
||||
assert record.payload is extra
|
||||
assert "payload" in json.loads(JsonFormatter().format(record))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra,scrubbed",
|
||||
(
|
||||
(("gpt-4o", "sk-1234567890abcdefghij"), ("gpt-4o", "REDACTED")),
|
||||
({"gpt-4o", "sk-1234567890abcdefghij"}, ["REDACTED", "gpt-4o"]),
|
||||
({"model": "gpt-4o", "key": "sk-1234567890abcdefghij"}, {"model": "gpt-4o", "key": "REDACTED"}),
|
||||
),
|
||||
ids=("tuple", "set", "dict"),
|
||||
)
|
||||
def test_extra_that_carried_a_secret_comes_back_scrubbed(monkeypatch, extra, scrubbed):
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.WARNING, "request sent")
|
||||
record.payload = extra
|
||||
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
rendered = JsonFormatter().format(record)
|
||||
|
||||
assert record.payload == scrubbed
|
||||
assert type(record.payload) is type(scrubbed)
|
||||
assert "sk-1234567890abcdefghij" not in rendered
|
||||
assert "REDACTED" in rendered
|
||||
|
||||
|
||||
class _AmbiguousArray:
|
||||
def __eq__(self, other: object) -> bool:
|
||||
raise ValueError("The truth value of an array with more than one element is ambiguous")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "array([1, 2])"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra,scrubbed",
|
||||
((_AmbiguousArray(), "array([1, 2])"), ({"weights": _AmbiguousArray()}, {"weights": "array([1, 2])"})),
|
||||
ids=("top_level", "nested"),
|
||||
)
|
||||
def test_extra_whose_equality_raises_still_comes_back_scrubbed(monkeypatch, extra, scrubbed):
|
||||
"""numpy arrays and torch tensors raise when compared for truth, so the keep-or-scrub
|
||||
decision must fall on the scrubbed copy instead of breaking the caller's log call."""
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.WARNING, "request sent")
|
||||
record.payload = extra
|
||||
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
|
||||
assert record.payload == scrubbed
|
||||
assert json.loads(JsonFormatter().format(record))["payload"] == scrubbed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra",
|
||||
(
|
||||
{1: "sk-1234567890abcdefghij"},
|
||||
{"model": {1: "sk-1234567890abcdefghij"}},
|
||||
_nest("sk-1234567890abcdefghij", 101),
|
||||
_RequestExtra(model="gpt-4o", attempt=2, api_key="sk-1234567890abcdefghij"),
|
||||
{"gpt-4o", "sk-1234567890abcdefghij", 1},
|
||||
),
|
||||
ids=("int_key", "nested_int_key", "deeper_than_safe_dumps", "dataclass_hidden_field", "unsortable_set"),
|
||||
)
|
||||
def test_extra_the_filter_cannot_fully_inspect_never_keeps_its_secret(monkeypatch, extra):
|
||||
"""Whatever safe_dumps would skip (non-string keys, anything past its depth limit,
|
||||
fields a repr hides) must not ride the original object past the redacted stamp."""
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.WARNING, "request sent")
|
||||
record.payload = extra
|
||||
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
rendered = JsonFormatter().format(record)
|
||||
|
||||
assert record.payload is not extra
|
||||
assert "sk-1234567890abcdefghij" not in str(record.payload)
|
||||
assert "sk-1234567890abcdefghij" not in rendered
|
||||
|
||||
|
||||
def test_secret_free_set_comes_back_as_its_json_shape(monkeypatch):
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.WARNING, "request sent")
|
||||
record.payload = {"gpt-4o", "gpt-4o-mini"}
|
||||
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
|
||||
assert record.payload == ["gpt-4o", "gpt-4o-mini"]
|
||||
assert json.loads(JsonFormatter().format(record))["payload"] == ["gpt-4o", "gpt-4o-mini"]
|
||||
|
||||
|
||||
def test_unscrubbed_record_is_still_redacted_by_the_formatter(monkeypatch):
|
||||
"""Records that never met SecretRedactionFilter (uvicorn's, in JSON mode) keep
|
||||
their formatter-side redaction."""
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.INFO, "key sk-1234567890abcdefghij")
|
||||
|
||||
assert "sk-1234567890abcdefghij" not in JsonFormatter().format(record)
|
||||
assert "sk-1234567890abcdefghij" not in CorrelationPlainFormatter(_PLAIN_LOG_FORMAT).format(record)
|
||||
|
||||
|
||||
def test_set_session_id_bounds_length():
|
||||
"""set_session_id() must bound length so an oversized caller-supplied value
|
||||
isn't repeated across every log line for the request."""
|
||||
|
|
|
|||
|
|
@ -7,18 +7,24 @@ and one has explicit zero-cost pricing in model_info, the other deployment
|
|||
should still use the built-in pricing.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from unittest.mock import patch
|
||||
from typing import Final
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE
|
||||
from litellm.litellm_core_utils.ptu_pricing import ptu_config_error
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
from litellm.utils import (
|
||||
_invalidate_model_cost_lowercase_map,
|
||||
|
|
@ -60,6 +66,324 @@ def _restore_model_cost_entries(original_entries):
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("initial_count", (1, DEFAULT_MAX_LRU_CACHE_SIZE + 1))
|
||||
async def test_discovered_limits_survive_deployment_growth_and_removal(
|
||||
initial_count: int, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
deployments: Final = tuple(
|
||||
Deployment(
|
||||
model_name=f"local-{index}",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="hosted_vllm/local-model", api_base="https://capacity.test/v1", api_key="local-key"
|
||||
),
|
||||
model_info=ModelInfo(id=f"capacity-{index}"),
|
||||
)
|
||||
for index in range(DEFAULT_MAX_LRU_CACHE_SIZE + 2)
|
||||
)
|
||||
router: Final = Router(model_list=[deployment.to_json() for deployment in deployments[:initial_count]])
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]})
|
||||
)
|
||||
) as client:
|
||||
handler.client = client
|
||||
await router.arefresh_model_info(client=handler)
|
||||
assert all(
|
||||
router.get_configured_token_limits(deployment.model_name) == (4096, 4096)
|
||||
for deployment in deployments[:initial_count]
|
||||
)
|
||||
for deployment in deployments[initial_count:]:
|
||||
router.add_deployment(deployment)
|
||||
await router._arefresh_deployment_model_info(router.model_list[-1], client=handler)
|
||||
assert all(
|
||||
router.get_configured_token_limits(deployment.model_name) == (4096, 4096) for deployment in deployments
|
||||
)
|
||||
for deployment in deployments[-2:]:
|
||||
router.delete_deployment(deployment.model_info.id or "")
|
||||
await router._arefresh_deployment_model_info(router.model_list[0], client=handler)
|
||||
assert all(
|
||||
router.get_configured_token_limits(deployment.model_name) == (4096, 4096) for deployment in deployments[:-2]
|
||||
)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
async def test_discovery_discards_metadata_for_a_replaced_deployment(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
router: Final = Router(model_list=[{
|
||||
"model_name": "local",
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/local-model",
|
||||
"api_base": "https://original.test/v1",
|
||||
"api_key": "local-key",
|
||||
},
|
||||
"model_info": {"id": "replaced-deployment"},
|
||||
}])
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.host == "original.test":
|
||||
router.upsert_deployment(Deployment(
|
||||
model_name="local",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="hosted_vllm/local-model",
|
||||
api_base="https://replacement.test/v1",
|
||||
api_key="local-key",
|
||||
),
|
||||
model_info=ModelInfo(id="replaced-deployment"),
|
||||
))
|
||||
return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 8192}]})
|
||||
assert request.url.host == "replacement.test"
|
||||
return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 2048}]})
|
||||
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
|
||||
handler.client = client
|
||||
await router._arefresh_deployment_model_info(router.model_list[0], client=handler)
|
||||
assert router.get_configured_token_limits("local") == (None, None)
|
||||
await router.arefresh_model_info(client=handler)
|
||||
assert router.get_configured_token_limits("local") == (2048, 2048)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
async def test_discovery_is_isolated_across_routers_and_reused_ids(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
first, second = tuple(
|
||||
Router(model_list=[{
|
||||
"model_name": "local",
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/local-model",
|
||||
"api_base": f"https://{host}.test/v1",
|
||||
"api_key": "local-key",
|
||||
},
|
||||
"model_info": {"id": "shared-discovery-id"},
|
||||
}])
|
||||
for host in ("first", "second")
|
||||
)
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.host == "unavailable.test":
|
||||
return httpx.Response(503)
|
||||
limit: Final = 8192 if request.url.host == "first.test" else 2048
|
||||
return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": limit}]})
|
||||
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
|
||||
handler.client = client
|
||||
await first.arefresh_model_info(client=handler)
|
||||
assert second.get_configured_token_limits("local") == (None, None)
|
||||
await second.arefresh_model_info(client=handler)
|
||||
assert first.get_discovered_model_info("shared-discovery-id")["max_input_tokens"] == 8192
|
||||
assert first.get_configured_token_limits("local") == (8192, 8192)
|
||||
assert second.get_configured_token_limits("local") == (2048, 2048)
|
||||
assert litellm.model_cost["shared-discovery-id"].get("max_input_tokens") is None
|
||||
first.upsert_deployment(Deployment(
|
||||
model_name="local",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="hosted_vllm/local-model",
|
||||
api_base="https://unavailable.test/v1",
|
||||
api_key="local-key",
|
||||
),
|
||||
model_info=ModelInfo(id="shared-discovery-id"),
|
||||
))
|
||||
assert first.get_configured_token_limits("local") == (None, None)
|
||||
await first.arefresh_model_info(client=handler)
|
||||
assert first.get_configured_token_limits("local") == (None, None)
|
||||
assert second.get_configured_token_limits("local") == (2048, 2048)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
async def test_discovery_refreshes_other_endpoints_while_one_is_pending(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
second_started: Final = asyncio.Event()
|
||||
router: Final = Router(model_list=[
|
||||
{
|
||||
"model_name": host,
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/local-model",
|
||||
"api_base": f"https://{host}.test/v1",
|
||||
"api_key": "local-key",
|
||||
},
|
||||
}
|
||||
for host in ("first", "second", "third")
|
||||
])
|
||||
|
||||
async def respond(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.host == "first.test":
|
||||
await second_started.wait()
|
||||
if request.url.host == "second.test":
|
||||
second_started.set()
|
||||
return httpx.Response(503)
|
||||
return httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 2048}]})
|
||||
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
|
||||
handler.client = client
|
||||
await asyncio.wait_for(router.arefresh_model_info(client=handler), timeout=2)
|
||||
assert router.get_configured_token_limits("first") == (2048, 2048)
|
||||
assert router.get_configured_token_limits("second") == (None, None)
|
||||
assert router.get_configured_token_limits("third") == (2048, 2048)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
async def test_discovered_limits_expire_after_the_last_successful_refresh(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
clock: Final = Mock(return_value=0.0)
|
||||
router: Final = Router(model_list=[{
|
||||
"model_name": "local",
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/local-model",
|
||||
"api_base": "https://expiry.test/v1",
|
||||
"api_key": "local-key",
|
||||
},
|
||||
"model_info": {"id": "expiring-discovery"},
|
||||
}])
|
||||
router._discovered_model_info_cache = InMemoryCache(clock=clock, default_ttl=2 * MODEL_INFO_REFRESH_SECONDS)
|
||||
responses: Final = iter((
|
||||
httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}),
|
||||
httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 8192}]}),
|
||||
httpx.Response(503),
|
||||
))
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(lambda request: next(responses))) as client:
|
||||
handler.client = client
|
||||
await router.arefresh_model_info(client=handler)
|
||||
clock.return_value = MODEL_INFO_REFRESH_SECONDS
|
||||
router.cache.in_memory_cache.flush_cache()
|
||||
await router.arefresh_model_info(client=handler)
|
||||
clock.return_value = 2 * MODEL_INFO_REFRESH_SECONDS + 1
|
||||
router.cache.in_memory_cache.flush_cache()
|
||||
await router.arefresh_model_info(client=handler)
|
||||
assert router.get_configured_token_limits("local") == (8192, 8192)
|
||||
group: Final = router.get_model_group_info("local")
|
||||
assert group is not None
|
||||
assert group.max_input_tokens == 8192
|
||||
clock.return_value = 3 * MODEL_INFO_REFRESH_SECONDS + 1
|
||||
await router.arefresh_model_info(client=handler)
|
||||
assert router.get_configured_token_limits("local") == (None, None)
|
||||
expired_group: Final = router.get_model_group_info("local")
|
||||
assert expired_group is not None
|
||||
assert expired_group.max_input_tokens is None
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", ("hosted_vllm", "openai", "openai_like", "text-completion-openai"))
|
||||
async def test_discovered_limits_are_isolated_overridable_and_refreshable(
|
||||
provider: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
upstream_limit: Final = iter((8192, 4096, 16384, 2048))
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/v1/models"
|
||||
assert request.headers["authorization"] == "Bearer local-key"
|
||||
return httpx.Response(200, json={"data": [{"id": "org/local-model", "max_model_len": next(upstream_limit)}]})
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "local",
|
||||
"litellm_params": {
|
||||
"model": f"{provider}/org/local-model",
|
||||
"api_base": f"https://{host}.test/v1",
|
||||
"api_key": "local-key",
|
||||
},
|
||||
"model_info": {"id": host, **overrides},
|
||||
}
|
||||
for host, overrides in (("one", {}), ("two", {"max_output_tokens": 512}))
|
||||
],
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
|
||||
handler.client = client
|
||||
await router.arefresh_model_info(client=handler)
|
||||
first: Final = router.get_router_model_info(id="one", deployment=None, received_model_name="local")
|
||||
second: Final = router.get_router_model_info(id="two", deployment=None, received_model_name="local")
|
||||
assert (first["max_input_tokens"], first["max_output_tokens"]) == (8192, 8192)
|
||||
assert (second["max_input_tokens"], second["max_output_tokens"]) == (4096, 512)
|
||||
group: Final = router.get_model_group_info("local")
|
||||
assert group is not None
|
||||
assert group.max_input_tokens == 8192
|
||||
listing: Final = router.get_model_listing_info("local")
|
||||
assert listing is not None
|
||||
assert listing.max_input_tokens == 8192
|
||||
assert router.get_configured_token_limits("local") == (8192, 8192)
|
||||
assert router._deployment_max_input_tokens("local", router.model_list[1]) == 4096
|
||||
allowed: Final = router._pre_call_checks(
|
||||
model="local", healthy_deployments=router.model_list, input="prompt", input_token_count=5000
|
||||
)
|
||||
assert [deployment["model_info"]["id"] for deployment in allowed] == ["one"]
|
||||
assert router.model_list[0]["model_info"].get("max_input_tokens") is None
|
||||
assert litellm.model_cost[f"{provider}/org/local-model"].get("max_input_tokens") is None
|
||||
router.cache.in_memory_cache.flush_cache()
|
||||
await router.arefresh_model_info(client=handler)
|
||||
refreshed: Final = router.get_model_group_info("local")
|
||||
assert refreshed is not None
|
||||
assert refreshed.max_input_tokens == 16384
|
||||
assert (
|
||||
router.get_router_model_info(id="two", deployment=None, received_model_name="local")["max_output_tokens"]
|
||||
== 512
|
||||
)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
async def test_discovery_preserves_input_overrides_and_survives_outages(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
responses: Final = iter((
|
||||
httpx.Response(200, json={"data": [{"id": "local-model", "max_model_len": 4096}]}),
|
||||
httpx.Response(503),
|
||||
))
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.host == "backend.test"
|
||||
assert request.headers["authorization"] == "Bearer local-key"
|
||||
assert request.headers["x-tenant"] == "tenant"
|
||||
return next(responses)
|
||||
|
||||
router: Final = Router(model_list=[
|
||||
{
|
||||
"model_name": "configured",
|
||||
"litellm_params": {
|
||||
"model": "hosted_vllm/local-model",
|
||||
"api_base": "https://backend.test/v1",
|
||||
"api_key": "unused-key",
|
||||
"extra_headers": {"authorization": "Bearer local-key", "X-Tenant": "tenant"},
|
||||
},
|
||||
"model_info": {"id": "configured", "max_input_tokens": 1024},
|
||||
},
|
||||
{
|
||||
"model_name": "byok",
|
||||
"litellm_params": {
|
||||
"model": "openai/local-model",
|
||||
"api_base": "https://caller.test/v1",
|
||||
"use_clientside_credentials": True,
|
||||
},
|
||||
},
|
||||
{"model_name": "default-openai", "litellm_params": {"model": "openai/local-model", "api_key": "unused"}},
|
||||
])
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
responder: Final = Mock(side_effect=respond)
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(responder)) as client:
|
||||
handler.client = client
|
||||
await router.arefresh_model_info(client=handler)
|
||||
assert router.get_configured_token_limits("configured") == (1024, 4096)
|
||||
router.cache.in_memory_cache.flush_cache()
|
||||
await router.arefresh_model_info(client=handler)
|
||||
assert router.get_configured_token_limits("configured") == (1024, 4096)
|
||||
assert router.get_configured_token_limits("byok") == (None, None)
|
||||
assert next(responses, None) is None
|
||||
assert responder.call_count == 2
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_should_not_pollute_shared_key_with_zero_cost_pricing():
|
||||
"""
|
||||
When deployment A has input_cost_per_token=0 and deployment B has no
|
||||
|
|
|
|||
|
|
@ -636,11 +636,13 @@ def test_aws_credential_redaction_catches_quoted_values():
|
|||
{"blob": {"authorization": f"Bearer {SECRET}"}},
|
||||
{"blob": [f"Bearer {SECRET}"]},
|
||||
{"blob": ({"nested": {"deep": SECRET}},)},
|
||||
{"master_key": "opaque-value-with-no-pattern"},
|
||||
),
|
||||
ids=("set", "dict", "list", "nested"),
|
||||
ids=("set", "dict", "list", "nested", "key_name"),
|
||||
)
|
||||
def test_json_formatter_redacts_non_string_extra_values(extra):
|
||||
"""SecretRedactionFilter only scrubs str attrs, so containers must be caught on render."""
|
||||
"""Container extras and key-named str extras must come out scrubbed, whichever of the
|
||||
filter and the formatter does the work."""
|
||||
buf = StringIO()
|
||||
handler = logging.StreamHandler(buf)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue