merge main into litellm_web_search_deployment_filter

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-10-16 22:17:19 +00:00
commit b7a1987c33
76 changed files with 6584 additions and 245 deletions

View file

@ -32,6 +32,7 @@ IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)")
DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
GLOB_CHARS = frozenset("*?")
DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"})
# Trees whose jobs are sharded with no catch-all bucket, so every child that holds
# tests has to be named by some shard or it runs nowhere. A child listed here is
@ -560,6 +561,17 @@ def _integration_github_files(runner: pathlib.Path) -> frozenset[str]:
return frozenset(ast.literal_eval(values))
def _owns(workflow: str, relative_path: str) -> bool:
owned_root: Final = DEDICATED_INTEGRATION_WORKFLOWS.get(workflow)
return owned_root is not None and relative_path.startswith(owned_root)
def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]:
return frozenset[str]().union(
*(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path))
)
def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
runner: Final = repo_root / "tests/integration/run.py"
if not runner.exists():
@ -611,15 +623,20 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
)
!= 1
)
gha_tokens: Final = _invoked_test_tokens(
scalar
for path in (repo_root / ".github/workflows").glob("*.y*ml")
for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
workflow_tokens: Final = MappingProxyType(
{
path.name: _invoked_test_tokens(_scalars(yaml.safe_load(path.read_text()), path.name))
for path in (repo_root / ".github/workflows").glob("*.y*ml")
}
)
gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values())
circle_owned: Final = (
frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files
)
findings: Final = (
tuple(
Finding(path, "integration contract is also selected by GitHub Actions")
for path in paths
Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions")
for path in sorted(circle_owned)
if any(_token_covers(token, path) for token in gha_tokens)
)
+ tuple(
@ -632,6 +649,11 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
for path in sorted(github_files)
if not (repo_root / path).is_file()
)
+ tuple(
Finding(path, "GitHub-owned integration contract runs outside a dedicated integration workflow")
for path in sorted(github_files)
if any(_token_covers(token, path) for token in _tokens_outside_owner(path, workflow_tokens))
)
)
browser_commands: Final = tuple(
scalar.value
@ -688,13 +710,9 @@ def main() -> int:
scalars = _all_scalars()
integration_paths, ownership_findings = _integration_ownership()
test_findings = (
_uncovered_tests(
allowlist,
_invoked_selections(scalars)
+ (Selection(included=integration_paths, ignored=frozenset()),),
)
+ ownership_findings
test_findings = _uncovered_tests(
allowlist,
_invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),),
)
dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
@ -711,13 +729,25 @@ def main() -> int:
test_findings,
"Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.",
)
if ownership_findings:
browser_remedy = (
" CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may "
"select them and the default Playwright config must exclude them."
if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings)
else ""
)
_report(
"integration contracts with the wrong CI owner",
ownership_findings,
"Follow tests/integration/AGENTS.md, which says which CI job owns each integration file." + browser_remedy,
)
if dockerfile_findings:
_report(
"Dockerfiles that no CI job builds",
dockerfile_findings,
"Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
)
if stale_findings or test_findings or dockerfile_findings:
if stale_findings or test_findings or ownership_findings or dockerfile_findings:
return 1
_write(

View file

@ -29,7 +29,7 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a
`tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md`
End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
@ -72,7 +72,7 @@ When referencing or running models (coding, QA'ing, writing docs, writing tests,
Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
If you're an internal contributor, when creating a new PR, the typical flow is to branch off the repository's current default branch and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
Internal contributors branch a new PR off the repository's current default branch, with a branch name prefixed with litellm_
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch

View file

@ -1416,6 +1416,7 @@ from .exceptions import (
InvalidRequestError,
BadRequestError,
ImageFetchError,
MalformedRequestFieldError as MalformedRequestFieldError,
VectorStoreSearchError,
NotFoundError,
PermissionDeniedError,

View file

@ -1859,6 +1859,7 @@ SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_L
PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25))
PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0))
PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0))
TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS: Final = 30
DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = get_env_int("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60) # 1 minute
PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)
RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, get_env_int("RESET_BUDGET_JOB_BATCH_SIZE", 500))

View file

@ -313,6 +313,10 @@ class VectorStoreSearchError(BadRequestError):
)
class MalformedRequestFieldError(BadRequestError):
pass
class UnprocessableEntityError(openai.UnprocessableEntityError):
def __init__(
self,

View file

@ -27,6 +27,7 @@ from litellm.constants import (
PROXY_REJECTED_BEFORE_ROUTING_KEY,
)
from litellm.exceptions import (
MalformedRequestFieldError,
validate_rate_limit_category,
validate_rate_limit_type,
)
@ -3104,7 +3105,11 @@ class PrometheusLogger(CustomLogger):
# On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"]
# into requested_model and leave deployment-scoped labels empty.
deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
deployment_selected: Final = (
bool(model_id)
and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
and not isinstance(exception, MalformedRequestFieldError)
)
if deployment_selected:
label_litellm_model_name = litellm_model_name
label_model_id = model_id

View file

@ -2300,6 +2300,13 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["combined_usage_object"] = usage
self.model_call_details["response_cost"] = response_cost
def attempt_scoped_copy(self) -> "Logging":
"""A copy with its own model_call_details, so logging one attempt of a request through it leaves
the state the request's other attempts share untouched."""
attempt: Final = copy.copy(self)
attempt.model_call_details = dict(self.model_call_details)
return attempt
def carry_failed_attempt_usage(self) -> None:
"""Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
own usage while the request's failure log still bills every attempt."""

View file

@ -858,9 +858,9 @@ class StandardBuiltInToolCostTracking:
@staticmethod
def get_web_search_options(kwargs: Mapping[str, object]) -> WebSearchOptions | None:
if "web_search_options" in kwargs:
web_search_options: Final = cast(WebSearchOptions, kwargs.get("web_search_options", {}))
return WebSearchOptions(**web_search_options)
web_search_options: Final = kwargs.get("web_search_options")
if isinstance(web_search_options, dict):
return WebSearchOptions(**cast(WebSearchOptions, web_search_options))
tools: Final = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
kwargs=kwargs, tool_type="web_search_preview"
@ -876,10 +876,9 @@ class StandardBuiltInToolCostTracking:
_get_web_search_options = get_web_search_options
@staticmethod
def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> list[object] | None:
if "tools" in kwargs:
return cast(list[object], kwargs.get("tools", []))
return None
def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> Sequence[object] | None:
tools: Final = kwargs.get("tools")
return cast(Sequence[object], tools) if isinstance(tools, (list, tuple)) else None
@staticmethod
def get_file_search_tool_call(kwargs: Mapping[str, object]) -> FileSearchTool | None:

View file

@ -0,0 +1,164 @@
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from functools import reduce
from typing import Annotated, Final, TypeAlias
from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
ROUTES_WITH_TOOLS: Final = frozenset(
{
"acompletion",
"aresponses",
"acompact_responses",
"anthropic_messages",
"agenerate_content",
"agenerate_content_stream",
"acreate_interaction",
}
)
TOOL_FIELDS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls", "toolConfig", "tool_config"})
_TOOLS: Final = TypeAdapter(list[object])
_FIELD_LIST: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
_JSON_OBJECT: Final = TypeAdapter(Json[dict[str, object]])
def _schema_as_read(value: object) -> object:
if not value:
return None
if not isinstance(value, str):
return value
try:
return _JSON_OBJECT.validate_python(value)
except ValidationError:
return value
_Schema: TypeAlias = Annotated[dict[str, object] | None, BeforeValidator(_schema_as_read)]
class _ToolFunction(TypedDict, total=False):
parameters: ReadOnly[_Schema]
class _ToolEntry(TypedDict, total=False):
function: ReadOnly[_ToolFunction | None]
parameters: ReadOnly[_Schema]
input_schema: ReadOnly[_Schema]
vector_store_ids: ReadOnly[list[str] | None]
_TOOL_ENTRY: Final = TypeAdapter(_ToolEntry)
_WEB_SEARCH_OPTIONS: Final = TypeAdapter(dict[str, object] | None, config=ConfigDict(strict=True))
@dataclass(frozen=True, slots=True)
class DropPolicy:
drop_params: bool
additional_drop_params: frozenset[str]
def drops(self, field: str) -> bool:
return self.drop_params or field in self.additional_drop_params
@dataclass(frozen=True, slots=True)
class MalformedField:
field: str
message: str
dropped: Callable[[Mapping[str, object]], Mapping[str, object]]
def _field_names(value: object) -> frozenset[str]:
try:
return frozenset(name for name in _FIELD_LIST.validate_python(value) if isinstance(name, str))
except ValidationError:
return frozenset()
def drop_policy(params: Mapping[str, object]) -> DropPolicy:
return DropPolicy(
drop_params=litellm.drop_params is True or normalize_drop_params(params.get("drop_params")) is True,
additional_drop_params=_field_names(params.get("additional_drop_params")),
)
def first_validation_error(e: ValidationError, prefix: tuple[str | int, ...] = ()) -> str:
error: Final = e.errors(include_url=False)[0]
return f"{'.'.join(str(part) for part in (*prefix, *error['loc']))}: {error['msg']}"
def _tool_entry_error(index: int, entry: object) -> str | None:
try:
_TOOL_ENTRY.validate_python(entry, strict=True)
except ValidationError as e:
return first_validation_error(e, prefix=("tools", index))
return None
def _without_tool_fields(data: Mapping[str, object]) -> Mapping[str, object]:
return {name: value for name, value in data.items() if name not in TOOL_FIELDS}
def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None:
tools: Final = data.get("tools")
if tools is None:
return None
try:
entries: Final = _TOOLS.validate_python(tools)
except ValidationError as e:
return MalformedField("tools", first_validation_error(e, prefix=("tools",)), _without_tool_fields)
entry_errors: Final = tuple(_tool_entry_error(index, entry) for index, entry in enumerate(entries))
first_error: Final = next((error for error in entry_errors if error is not None), None)
if first_error is None:
return None
kept: Final = [entry for entry, error in zip(entries, entry_errors) if error is None]
return MalformedField(
"tools", first_error, lambda fields: {**fields, "tools": kept} if kept else _without_tool_fields(fields)
)
def _without_web_search_options(data: Mapping[str, object]) -> Mapping[str, object]:
return {name: value for name, value in data.items() if name != "web_search_options"}
def _malformed_web_search_options(data: Mapping[str, object]) -> MalformedField | None:
try:
_WEB_SEARCH_OPTIONS.validate_python(data.get("web_search_options"))
except ValidationError as e:
return MalformedField(
"web_search_options",
first_validation_error(e, prefix=("web_search_options",)),
_without_web_search_options,
)
return None
def resolve_tool_fields(
data: Mapping[str, object], candidates: Callable[[], tuple[DropPolicy, ...]]
) -> Mapping[str, object] | MalformedField:
checked: Final = (_malformed_tools(data), _malformed_web_search_options(data))
malformed: Final = tuple(field for field in checked if field is not None)
if not malformed:
return data
policies: Final = candidates()
rejected: Final = next(
(field for field in malformed if not any(policy.drops(field.field) for policy in policies)), None
)
if rejected is not None:
return rejected
return reduce(
lambda resolved, field: field.dropped(resolved),
(field for field in malformed if all(policy.drops(field.field) for policy in policies)),
data,
)
def resolve_deployment_tool_fields(
kwargs: Mapping[str, object], route: object
) -> Mapping[str, object] | MalformedField:
if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS:
return kwargs
return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),))

View file

@ -45,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
list_field,
merge_guardrailed_scoped_messages,
merge_returned_tools_into_request_tools,
scoped_structured_message_indices,
@ -924,10 +925,10 @@ class AnthropicMessagesHandler(BaseTranslation):
images=(),
)
def extract_request_tool_names(self, data: dict) -> list[str]:
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
"""Extract every tool name in an Anthropic messages request: tools[].name, plus
tools[].function.name for OpenAI-format tools the bridge forwards verbatim."""
return [name for tool in data.get("tools") or [] for name in anthropic_tool_names(tool)]
return [name for tool in list_field(data, "tools") for name in anthropic_tool_names(tool)]
@classmethod
def _extract_input_text_and_images(

View file

@ -2,6 +2,7 @@ import asyncio
import json
import re
from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence
from contextvars import ContextVar
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable
@ -352,6 +353,22 @@ async def aclose_if_supported(stream: object) -> None:
await stream.aclose()
_CLOSING_SUPERSEDED_ATTEMPT: Final[ContextVar[bool]] = ContextVar("closing_superseded_attempt", default=False)
async def aclose_superseded_attempt(stream: object) -> None:
"""Close an attempt a retry or fallback is taking over, so its end-of-stream logging records a failure."""
token: Final = _CLOSING_SUPERSEDED_ATTEMPT.set(True)
try:
await aclose_if_supported(stream)
finally:
_CLOSING_SUPERSEDED_ATTEMPT.reset(token)
def closing_superseded_attempt() -> bool:
return _CLOSING_SUPERSEDED_ATTEMPT.get()
_RESPONSE_HEADERS_ADAPTER: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])

View file

@ -4,7 +4,7 @@ import json
from collections.abc import Callable, Iterator, Mapping, Sequence
from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles
from pydantic import BaseModel
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
@ -292,6 +292,17 @@ def openai_tool_name(tool: object) -> str | None:
return flat_name if isinstance(flat_name, str) else None
_FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True))
_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
def list_field(data: object, key: str) -> Sequence[object]:
try:
return _ITEMS.validate_python(_FIELDS.validate_python(data).get(key))
except ValidationError:
return ()
def anthropic_tool_names(tool: object) -> tuple[str, ...]:
"""Every name a /v1/messages tool dict can act under: the flat Anthropic ``name`` plus
``function.name`` for OpenAI-format tools the bridge forwards verbatim. Allowlist checks

View file

@ -39,6 +39,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
list_field,
merge_guardrailed_scoped_messages,
merge_returned_tools_into_request_tools,
openai_tool_name,
@ -156,7 +157,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
inputs["structured_messages"] = [structured_messages[index] for index in scoped_message_indices]
# Pass tools (function definitions) to the guardrail
tools: Final = data.get("tools")
if tools and not scan_only_tool_results:
if isinstance(tools, list) and tools and not scan_only_tool_results:
inputs["tools"] = tools
# Include model information if available
model: Final = data.get("model")
@ -258,19 +259,19 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
return "no scannable content after message scoping"
return None if images else "no scannable content"
def extract_request_tool_names(self, data: dict) -> list[str]:
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
"""Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name,
and the tools a replayed tool_search_output loads)."""
names: Final[list[str]] = []
for tool in data.get("tools") or []:
for tool in list_field(data, "tools"):
if isinstance(tool, dict) and tool.get("type") == "function":
fn = tool.get("function")
if isinstance(fn, dict) and fn.get("name"):
names.append(str(fn["name"]))
for fn in data.get("functions") or []:
for fn in list_field(data, "functions"):
if isinstance(fn, dict) and fn.get("name"):
names.append(str(fn["name"]))
return [*names, *replayed_tool_search_tool_names(cast(object, data))] # cast-ok: upcast, read field by field
return [*names, *replayed_tool_search_tool_names(data)]
def _extract_inputs(
self,

View file

@ -55,6 +55,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_responses_stream_usage,
effective_skip_system_message_for_guardrail,
list_field,
merge_guardrailed_scoped_messages,
role_out_of_guardrail_scope,
scoped_structured_message_indices,
@ -764,11 +765,11 @@ class OpenAIResponsesHandler(BaseTranslation):
data.get("input"), data.get("instructions"), structured_messages, merged
)
def extract_request_tool_names(self, data: dict) -> list[str]:
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
"""Extract tool names from Responses API request (tools[].name for function
and custom, tools[].server_label for mcp)."""
names: Final[list[str]] = []
for tool in data.get("tools") or []:
for tool in list_field(data, "tools"):
if not isinstance(tool, dict):
continue
if tool.get("type") in ("function", "custom") and tool.get("name"):

View file

@ -55,6 +55,7 @@ from litellm.proxy._experimental.mcp_server.catalog import (
)
from litellm.proxy._experimental.mcp_server.contracts import OperationContext
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.types.mcp import MCP_LEGACY_VERSIONS
from litellm.types.mcp_server.mcp_server_manager import MCPServer
@ -300,6 +301,29 @@ async def _listing(event: ServerEvent, context: OperationContext) -> str | None:
raise MCPError(code=-32603, message="Subscription catalog exceeds the pagination limit")
async def _enforce_rate_limit(context: OperationContext, server: MCPServer) -> None:
await operations._enforce_mcp_server_rate_limit(context.user_api_key_auth, server) # pyright: ignore[reportPrivateUsage] # same admission policy as other gateway operations
async def _unmetered(_: OperationContext, __: MCPServer) -> None:
return None
async def _admitted() -> ProxyRateLimitError | None:
return None
@asynccontextmanager
async def _listener_admission(targets: tuple[SubscriptionTarget, ...]) -> AsyncGenerator[None]:
scope: Final = operations._mcp_server_admission_memo.set( # pyright: ignore[reportPrivateUsage] # the listener's own catalog refreshes reuse its admission
{target.server.server_id: asyncio.create_task(_admitted()) for target in targets}
)
try:
yield
finally:
operations._mcp_server_admission_memo.reset(scope) # pyright: ignore[reportPrivateUsage] # restore the caller's admission scope
async def _recheck_authorization(check: Callable[[], Awaitable[OperationContext]]) -> None:
while True:
await asyncio.sleep(15)
@ -361,7 +385,7 @@ class GatewaySubscriptions:
if ctx.protocol_version != "2026-07-28":
raise MCPError(code=-32601, message="Subscriptions require the modern MCP protocol")
async with self._capacity(owner):
return await self._listen(ctx, params, authorize)
return await self._listen(ctx, params, authorize, _enforce_rate_limit)
@asynccontextmanager
async def _capacity(self, owner: str) -> AsyncGenerator[None]:
@ -414,6 +438,7 @@ class GatewaySubscriptions:
ctx,
SubscriptionFilter(tools_list_changed=True, prompts_list_changed=True, resources_list_changed=True),
authorization,
_unmetered,
)
if (ready := self._initializing.get(key)) is not None:
ready.set()
@ -427,6 +452,7 @@ class GatewaySubscriptions:
ctx: ServerRequestContext,
notifications: SubscriptionFilter,
authorization: SubscriptionAuthorization,
admit: Callable[[OperationContext, MCPServer], Awaitable[None]],
scope: anyio.CancelScope | None = None,
*,
task_status: TaskStatus[None],
@ -443,7 +469,7 @@ class GatewaySubscriptions:
async with self._capacity(authorization.owner):
await self._listen(
ctx, SubscriptionsListenRequestParams(notifications=notifications), authorization, ready
ctx, SubscriptionsListenRequestParams(notifications=notifications), authorization, admit, ready
)
async def legacy_resource(self, ctx: ServerRequestContext, uri: str, *, subscribe: bool) -> None:
@ -464,6 +490,7 @@ class GatewaySubscriptions:
ctx,
SubscriptionFilter(resource_subscriptions=[uri]),
state.authorization,
_enforce_rate_limit,
scope,
)
state.resources = {**state.resources, uri: scope}
@ -476,6 +503,7 @@ class GatewaySubscriptions:
ctx: ServerRequestContext,
params: SubscriptionsListenRequestParams,
authorize: Callable[[], Awaitable[OperationContext]],
admit: Callable[[OperationContext, MCPServer], Awaitable[None]],
ready: Callable[[SubscriptionFilter], None] | None = None,
) -> SubscriptionsListenResult:
context: Final = await authorize()
@ -501,7 +529,7 @@ class GatewaySubscriptions:
return fresh
async def relay(target: SubscriptionTarget, *, task_status: TaskStatus[SubscriptionFilter]) -> None:
await operations._enforce_mcp_server_rate_limit(context.user_api_key_auth, target.server) # pyright: ignore[reportPrivateUsage] # same admission policy as other gateway operations
await admit(context, target.server)
legacy: Final = LegacyNotifications()
@ -567,7 +595,7 @@ class GatewaySubscriptions:
finally:
self._upstream_slots.release()
async with anyio.create_task_group() as group:
async with _listener_admission(targets), anyio.create_task_group() as group:
honored: Final = tuple(
[SubscriptionFilter.model_validate(await group.start(relay, target)) for target in targets]
)

View file

@ -14261,6 +14261,19 @@
"description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.",
"title": "Fail On Error"
},
"forward_messages_as_sent": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"default": false,
"description": "Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can carry provider-native content blocks and keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with undeclared keys removed.",
"title": "Forward Messages As Sent"
},
"guard_name": {
"anyOf": [
{
@ -16612,6 +16625,19 @@
"description": "HTTP timeout in seconds for checking attached files. Default: 10.",
"title": "File Guardrail Timeout"
},
"forward_messages_as_sent": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"default": false,
"description": "Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can carry provider-native content blocks and keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with undeclared keys removed.",
"title": "Forward Messages As Sent"
},
"gateway_name": {
"anyOf": [
{

View file

@ -6,9 +6,9 @@ from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from fastapi.responses import JSONResponse
from pydantic import JsonValue
from pydantic import JsonValue, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -18,6 +18,7 @@ from litellm.anthropic_interface.exceptions import (
AnthropicExceptionMapping,
)
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.litellm_core_utils.tool_fields import first_validation_error
from litellm.llms.anthropic.pass_through.context_management import (
AnthropicContextManagementError,
)
@ -32,6 +33,7 @@ from litellm.proxy.common_request_processing import (
log_llm_api_exception,
proxy_exception_from_http_exception,
proxy_exception_from_route_error,
resolve_counted_tool_fields,
resolve_litellm_call_id,
)
from litellm.proxy.common_utils.error_body_call_id import error_body_call_id
@ -47,6 +49,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
with_litellm_call_id,
)
from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli
from litellm.router import Router
router: Final = APIRouter()
@ -91,6 +94,31 @@ def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSO
return JSONResponse(status_code=status_code, content=content, headers=exc.headers)
_TOKEN_COUNT_FIELDS: Final = ("model", "messages", "tools", "system")
def _token_count_request(fields: Mapping[str, object]) -> TokenCountRequest | ProxyException:
try:
return TokenCountRequest.model_validate({name: fields[name] for name in _TOKEN_COUNT_FIELDS if name in fields})
except ValidationError as e:
return ProxyException(
message=first_validation_error(e),
type="invalid_request_error",
param=str(e.errors(include_url=False)[0]["loc"][0]),
code=status.HTTP_400_BAD_REQUEST,
)
def _token_count_fields(
data: Mapping[str, object], auto_drop_params: bool, llm_router: Router | None, team_id: str | None
) -> Mapping[str, object] | ProxyException:
fields: Final = {
name: data[name] for name in (*_TOKEN_COUNT_FIELDS, "drop_params", "additional_drop_params") if name in data
}
resolving: Final = {**fields, "drop_params": True} if auto_drop_params else fields
return resolve_counted_tool_fields(resolving, llm_router, team_id)
def _strip_total_tokens_from_anthropic_response(response: object) -> None:
"""Remove the OpenAI-flavored `usage.total_tokens` field that LiteLLM
injects into Anthropic /v1/messages responses.
@ -351,7 +379,7 @@ async def count_tokens(
Returns: {"input_tokens": <number>}
"""
from litellm.proxy.proxy_server import count_request_tokens, proxy_config
from litellm.proxy.proxy_server import count_request_tokens, llm_router, proxy_config
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
try:
@ -368,26 +396,21 @@ async def count_tokens(
if not messages:
raise HTTPException(status_code=400, detail="messages parameter is required")
# Create TokenCountRequest for the internal endpoint
from litellm.proxy._types import TokenCountRequest
token_request: Final = TokenCountRequest(
model=model_name,
messages=messages,
tools=data.get("tools"),
system=data.get("system"),
auto_drop_params: Final = should_auto_drop_params_for_agentic_cli(
request.headers.get("user-agent", ""), "drop_params" in data, proxy_config
)
token_fields: Final = _token_count_fields(data, auto_drop_params, llm_router, user_api_key_dict.team_id)
token_request: Final = (
token_fields if isinstance(token_fields, ProxyException) else _token_count_request(token_fields)
)
if isinstance(token_request, ProxyException):
return _anthropic_error_json_response(token_request, request)
token_response: Final = await count_request_tokens(
request=token_request,
call_endpoint=True,
messages_format="anthropic",
**_count_tokens_body_options(
data,
should_auto_drop_params_for_agentic_cli(
request.headers.get("user-agent", ""), "drop_params" in data, proxy_config
),
),
**_count_tokens_body_options(data, auto_drop_params),
)
except HTTPException as e:
return _anthropic_error_json_response(proxy_exception_from_http_exception(e, {}), request)

View file

@ -80,6 +80,7 @@ from litellm.litellm_core_utils.served_output_texts import (
from litellm.litellm_core_utils.streaming_handler import (
backfill_missing_cache_usage_fields,
)
from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields
from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import (
can_key_call_resolved_model,
@ -129,6 +130,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
)
from litellm.router import Router
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.router import RouterRateLimitError
@ -633,6 +635,63 @@ def serialize_http_exception_detail(
return str(detail), None
_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True))
_UNSEEN_DEPLOYMENT: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset())
def _is_auto_router(params: Mapping[str, object]) -> bool:
model: Final = params.get("model")
return isinstance(model, str) and model.startswith(AUTO_ROUTER_MODEL_PREFIX)
def _named_deployments(llm_router: Router, model: str, team_id: str | None) -> tuple[Mapping[str, object], ...]:
by_id: Final = None if model in llm_router.model_names else llm_router.get_deployment(model_id=model)
if by_id is not None:
return (_REQUEST_FIELDS.validate_python(by_id.litellm_params.model_dump(exclude_none=True)),)
group: Final = llm_router.get_model_from_alias(model=model) or model
if group not in llm_router.model_names or llm_router.get_model_list_from_routing_groups(model_name=group):
return ()
deployments: Final = tuple(
_REQUEST_FIELDS.validate_python(deployment["litellm_params"])
for deployment in llm_router.get_all_deployments(model_name=group, team_id=team_id)
)
return () if any(_is_auto_router(params) for params in deployments) else deployments
def _router_defaults(llm_router: Router | None) -> Mapping[str, object]:
configured: Final[object] = getattr(llm_router, "default_litellm_params", None)
if configured is None:
return {}
return {name: value for name, value in _REQUEST_FIELDS.validate_python(configured).items() if value is not None}
def _counted_drop_policies(
data: Mapping[str, object], llm_router: Router | None, team_id: str | None
) -> tuple[DropPolicy, ...]:
requested: Final = {**_router_defaults(llm_router), **data}
model: Final = data.get("model")
deployments: Final = (
_named_deployments(llm_router, model, team_id) if llm_router is not None and isinstance(model, str) else ()
)
if not deployments:
return (drop_policy(requested), _UNSEEN_DEPLOYMENT)
return tuple(drop_policy({**deployment, **requested}) for deployment in deployments)
def resolve_counted_tool_fields(
data: Mapping[str, object], llm_router: Router | None, team_id: str | None
) -> Mapping[str, object] | ProxyException:
resolved: Final = resolve_tool_fields(data, lambda: _counted_drop_policies(data, llm_router, team_id))
if isinstance(resolved, MalformedField):
return ProxyException(
message=resolved.message,
type="invalid_request_error",
param=resolved.field,
code=status.HTTP_400_BAD_REQUEST,
)
return resolved
def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, str]) -> ProxyException:
raw_detail: Final = attribute_of(exc, "detail", str(exc))
message, structured_fields = serialize_http_exception_detail(raw_detail)

View file

@ -32,6 +32,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
additional_provider_specific_params=getattr(litellm_params, "additional_provider_specific_params", {}),
unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"),
fail_on_error=getattr(litellm_params, "fail_on_error", True),
forward_messages_as_sent=getattr(litellm_params, "forward_messages_as_sent", False),
extra_headers=getattr(litellm_params, "extra_headers", None),
guardrail_name=guardrail.get("guardrail_name", ""),
event_hook=litellm_params.mode,

View file

@ -7,10 +7,12 @@
import fnmatch
import os
from collections.abc import Mapping, Sequence
from collections.abc import Callable, Mapping, Sequence
from typing import TYPE_CHECKING, Final, Literal, Optional
import httpx
from pydantic import JsonValue, TypeAdapter, ValidationError
from pydantic_core import PydanticSerializationError
from litellm._logging import verbose_proxy_logger
from litellm._version import version as litellm_version
@ -38,6 +40,10 @@ if TYPE_CHECKING:
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
GUARDRAIL_NAME: Final = "generic_guardrail_api"
_CHAT_ROW: Final[TypeAdapter[AllMessageValues]] = TypeAdapter(AllMessageValues)
_SENT_ROW: Final = TypeAdapter(Mapping[str, object])
_SENT_ROWS: Final = TypeAdapter(Sequence[Mapping[str, object]])
_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
# Headers whose values are forwarded as-is (case-insensitive). Glob patterns supported (e.g. x-stainless-*, x-litellm*).
_HEADER_VALUE_ALLOWLIST: Final = frozenset(
@ -150,23 +156,65 @@ def _extract_inbound_headers(
return None
def _is_contained_in(part: JsonValue, whole: JsonValue) -> bool:
pending: Final[list[tuple[JsonValue, JsonValue]]] = [(part, whole)] # mutable-ok: worklist, recursion is banned
while pending:
match pending.pop():
case dict() as part_dict, dict() as whole_dict:
if not part_dict.keys() <= whole_dict.keys():
return False
pending.extend((value, whole_dict[key]) for key, value in part_dict.items())
case list() as part_list, list() as whole_list:
if len(part_list) != len(whole_list):
return False
pending.extend(zip(part_list, whole_list, strict=True))
case part_leaf, whole_leaf if part_leaf != whole_leaf:
return False
case _:
pass
return True
def _is_shown_row(shown: AllMessageValues, returned: AllMessageValues) -> bool:
return returned == shown
def _is_echo(original: AllMessageValues, returned: AllMessageValues) -> bool:
"""A server that parses rows with its own message types and echoes them back drops the
keys those types do not declare. That is still an echo as long as nothing changed and
every key the chat completion types declare is kept."""
if returned == original:
return True
try:
sent: Final[JsonValue] = _JSON.validate_json(_SENT_ROW.dump_json(original))
declared: Final[JsonValue] = _JSON.validate_json(
_CHAT_ROW.dump_json(_CHAT_ROW.validate_python(original), warnings="error")
)
echoed: Final[JsonValue] = _JSON.validate_python(returned)
except (ValidationError, PydanticSerializationError):
return False
return _is_contained_in(echoed, sent) and _is_contained_in(declared, echoed)
def _structured_rows_to_write_back(
original_rows: Sequence[AllMessageValues] | None,
shown_rows: Sequence[AllMessageValues] | None,
compared_rows: Sequence[AllMessageValues] | None,
returned_rows: Sequence[AllMessageValues],
is_echo: Callable[[AllMessageValues, AllMessageValues], bool],
) -> tuple[AllMessageValues, ...] | None:
"""The request model drops row keys its message types do not declare, so a
row the server echoes back verbatim is restored to the original row object.
A server that echoes every row back unchanged has not rewritten anything
per row, so its answer is read from texts, as it was before rows could be
returned at all."""
if original_rows is None or shown_rows is None or len(returned_rows) != len(original_rows):
"""A row the server echoes back is restored to the original row object. A server
that echoes every row back has not rewritten anything per row, so its answer is
read from texts, as it was before rows could be returned at all."""
if original_rows is None or compared_rows is None or len(returned_rows) != len(original_rows):
return tuple(returned_rows)
if all(returned == shown for shown, returned in zip(shown_rows, returned_rows)):
echoed: Final = tuple(
is_echo(compared, returned) for compared, returned in zip(compared_rows, returned_rows, strict=True)
)
if all(echoed):
return None
return tuple(
original if returned == shown else returned
for original, shown, returned in zip(original_rows, shown_rows, returned_rows)
original if was_echoed else returned
for original, returned, was_echoed in zip(original_rows, returned_rows, echoed, strict=True)
)
@ -200,6 +248,7 @@ class GenericGuardrailAPI(CustomGuardrail):
additional_provider_specific_params: Mapping[str, object] | None = None,
unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed",
fail_on_error: bool | None = True,
forward_messages_as_sent: bool | None = False,
extra_headers: list | None = None,
streaming_end_of_stream_only: bool | None = None,
streaming_sampling_rate: int | None = None,
@ -235,6 +284,7 @@ class GenericGuardrailAPI(CustomGuardrail):
self.unreachable_fallback: Literal["fail_closed", "fail_open"] = unreachable_fallback
self.fail_on_error: bool = True if fail_on_error is None else fail_on_error
self.forward_messages_as_sent: bool = bool(forward_messages_as_sent)
# Read by UnifiedLLMGuardrails.async_post_call_streaming_iterator_hook
# via getattr(guardrail_to_apply, "streaming_*", default).
@ -316,15 +366,17 @@ class GenericGuardrailAPI(CustomGuardrail):
http_status_code: int | None = None,
) -> GenericGuardrailAPIInputs:
status_suffix: Final = f" http_status_code={http_status_code}" if http_status_code else ""
error_type: Final = "HTTPStatusError" if isinstance(error, httpx.HTTPStatusError) else type(error).__name__
verbose_proxy_logger.critical(
"Generic Guardrail API error (fail-open). Proceeding without guardrail.%s "
"guardrail_name=%s api_base=%s input_type=%s litellm_call_id=%s litellm_trace_id=%s",
"guardrail_name=%s api_base=%s input_type=%s litellm_call_id=%s litellm_trace_id=%s error_type=%s",
status_suffix,
getattr(self, "guardrail_name", None),
getattr(self, "api_base", None),
input_type,
getattr(logging_obj, "litellm_call_id", None) if logging_obj else None,
getattr(logging_obj, "litellm_trace_id", None) if logging_obj else None,
error_type,
exc_info=error,
)
# Keep flow going - treat as action=NONE (no modifications)
@ -339,10 +391,23 @@ class GenericGuardrailAPI(CustomGuardrail):
headers.update(self.headers)
return headers
def _request_body(
self,
guardrail_request: GenericGuardrailAPIRequest,
structured_messages: Sequence[AllMessageValues] | None,
) -> Mapping[str, object]:
body: Final[Mapping[str, object]] = guardrail_request.model_dump(mode="json")
if not self.forward_messages_as_sent or structured_messages is None:
return body
return {
**body,
"structured_messages": _SENT_ROWS.dump_python(_SENT_ROWS.validate_python(structured_messages), mode="json"),
}
def _build_guardrail_return_inputs(
self,
*,
texts: list,
texts: list[str],
images: list[str] | None,
tools: list[ChatCompletionToolParam] | None,
structured_messages: Sequence[AllMessageValues] | None,
@ -361,8 +426,13 @@ class GenericGuardrailAPI(CustomGuardrail):
return_inputs["tools"] = guardrail_response.tools
elif tools:
return_inputs["tools"] = tools
compared_rows, is_echo = (
(structured_messages, _is_echo) if self.forward_messages_as_sent else (shown_messages, _is_shown_row)
)
rows_to_write_back: Final = (
_structured_rows_to_write_back(structured_messages, shown_messages, guardrail_response.structured_messages)
_structured_rows_to_write_back(
structured_messages, compared_rows, guardrail_response.structured_messages, is_echo
)
if guardrail_response.structured_messages
else None
)
@ -465,7 +535,7 @@ class GenericGuardrailAPI(CustomGuardrail):
litellm_version=litellm_version,
images=images,
tools=([GuardrailToolParam.model_validate(t) for t in tools] if tools else None),
structured_messages=structured_messages,
structured_messages=None if self.forward_messages_as_sent else structured_messages,
tool_calls=tool_calls,
additional_provider_specific_params=additional_params,
input_type=input_type,
@ -478,7 +548,7 @@ class GenericGuardrailAPI(CustomGuardrail):
# Use mode="json" to ensure all iterables are converted to lists
response: Final = await self.async_handler.post(
url=self.api_base,
json=guardrail_request.model_dump(mode="json"),
json=dict(self._request_body(guardrail_request, structured_messages)),
headers=headers,
timeout=self.timeout,
)

View file

@ -6,23 +6,23 @@ knowledge lives in one place. Uses guardrail translation handlers where availabl
with standalone extractors for generate_content and MCP.
"""
from collections.abc import Mapping
from typing import Any, Final
from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
from litellm.llms import load_guardrail_translation_mappings
from litellm.llms.base_llm.guardrail_translation.utils import list_field
from litellm.types.utils import CallTypes
# Call types that have no guardrail translation handler; we use standalone extractors
STANDALONE_EXTRACTORS: Final[dict[str, Any]] = {}
def _extract_generate_content_tool_names(data: dict) -> list[str]:
def _extract_generate_content_tool_names(data: Mapping[str, object]) -> list[str]:
"""Google generateContent: tools[].functionDeclarations[].name"""
names: Final[list[str]] = []
for tool in data.get("tools") or []:
if not isinstance(tool, dict):
continue
for decl in tool.get("functionDeclarations") or []:
for tool in list_field(data, "tools"):
for decl in list_field(tool, "functionDeclarations"):
if isinstance(decl, dict) and decl.get("name"):
names.append(str(decl["name"]))
return names

View file

@ -425,7 +425,7 @@ async def add_new_user_to_default_team(
user_id: str,
user_email: str | None,
user_api_key_dict: UserAPIKeyAuth,
teams: list[str] | list[NewUserRequestTeam],
teams: Sequence[str | NewUserRequestTeam],
prisma_client: "PrismaClient",
):
tasks: Final[list[Awaitable[object]]] = []

View file

@ -28,7 +28,7 @@ from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError,
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.constants import LITELLM_PROXY_ADMIN_NAME
from litellm.constants import LITELLM_PROXY_ADMIN_NAME, TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.ptu_pricing import (
@ -1881,7 +1881,7 @@ async def delete_team_models(
"""
deleted_model_ids: Final[list[str]] = []
transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db)
async with transaction_client.tx() as tx:
async with transaction_client.tx(timeout=datetime.timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS)) as tx:
for team_id in team_ids:
rows = await _get_team_deployments(team_id, prisma_client, table=tx.litellm_proxymodeltable)
model_ids = [row.model_id for row in rows]

View file

@ -17,7 +17,7 @@ import traceback
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Set as AbstractSet
from dataclasses import dataclass
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from itertools import chain
from types import MappingProxyType
from typing import (
@ -42,6 +42,7 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.caching.redis_batch import active_request_redis_batches
from litellm.constants import TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS
from litellm.integrations.prometheus import PrometheusLogger
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy._types import (
@ -4674,7 +4675,7 @@ async def delete_team(
# or is still waiting on the lock, in which case its own re-read happens after this commits
# and sees the row gone before it writes anything.
delete_filter: Final[_TeamIdInFilter] = {"team_id": {"in": data.team_ids}}
async with prisma_client.tx() as tx:
async with prisma_client.tx(timeout=timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS)) as tx:
for team_id in sorted(data.team_ids):
await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id)
await tx.litellm_teamtable.delete_many(where=delete_filter)

View file

@ -89,6 +89,7 @@ from litellm.proxy._types import (
Member,
NewTeamRequest,
NewUserRequest,
NewUserRequestTeam,
NewUserResponse,
ProxyErrorTypes,
ProxyException,
@ -118,7 +119,11 @@ from litellm.proxy.common_utils.html_forms.jwt_display_template import (
from litellm.proxy.common_utils.html_forms.ui_login import build_ui_login_form
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.proxy.management_endpoints.internal_user_endpoints import new_user
from litellm.proxy.management_endpoints.internal_user_endpoints import (
add_new_user_to_default_team,
check_if_default_team_set,
new_user,
)
from litellm.proxy.management_endpoints.sso import CustomMicrosoftSSO, LiteLLMGoogleSSO, create_generic_sso_provider
from litellm.proxy.management_endpoints.sso.id_jag_assertion_capture import (
id_jag_assertion_capture_gap,
@ -2825,6 +2830,52 @@ async def insert_sso_user(
return response
DefaultTeamJoin: TypeAlias = Callable[
[LiteLLM_UserTable, Sequence[str | NewUserRequestTeam], PrismaClient], Awaitable[None]
]
async def _join_default_teams_as_admin(
user: LiteLLM_UserTable, teams: Sequence[str | NewUserRequestTeam], prisma_client: PrismaClient
) -> None:
await add_new_user_to_default_team(
user_id=user.user_id,
user_email=user.user_email,
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
teams=teams,
prisma_client=prisma_client,
)
def _default_teams_not_joined(user: LiteLLM_UserTable) -> tuple[str | NewUserRequestTeam, ...]:
joined: Final = frozenset(user.teams)
return tuple(
team
for team in check_if_default_team_set() or ()
if (team if isinstance(team, str) else team.team_id) not in joined
)
async def _insert_sso_user_or_find_concurrent_insert(
insert: Awaitable[NewUserResponse],
user_id: str | None,
prisma_client: PrismaClient,
join_default_teams: DefaultTeamJoin = _join_default_teams_as_admin,
) -> NewUserResponse | LiteLLM_UserTable:
try:
return await insert
except Exception:
users: Final = UserRepository(prisma_client, use_writer=True)
concurrent_insert: Final = None if user_id is None else await users.find_by_id(user_id)
if concurrent_insert is None:
raise
unjoined: Final = _default_teams_not_joined(concurrent_insert)
if not unjoined:
return concurrent_insert
await join_default_teams(concurrent_insert, unjoined, prisma_client)
return await users.find_by_id(concurrent_insert.user_id) or concurrent_insert
@router.get(
"/sso/get/ui_settings",
tags=["experimental"],
@ -3471,9 +3522,10 @@ class SSOAuthenticationHandler:
else:
verbose_proxy_logger.info("user not in DB, inserting user into LiteLLM DB")
# user not in DB, insert User into LiteLLM DB
user_info = await insert_sso_user(
result_openid=result,
user_defined_values=user_defined_values,
user_info = await _insert_sso_user_or_find_concurrent_insert(
insert=insert_sso_user(result_openid=result, user_defined_values=user_defined_values),
user_id=None if user_defined_values is None else user_defined_values.get("user_id"),
prisma_client=prisma_client,
)
return user_info
except Exception as e:

View file

@ -10,12 +10,14 @@ from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import ANTHROPIC_BATCHES_ROUTE
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_content_from_model_response,
)
from litellm.llms.anthropic import get_anthropic_config
from litellm.llms.anthropic.common_utils import anthropic_error_frame_exception
from litellm.llms.anthropic.pass_through.stream_assembly import (
build_complete_streaming_response,
build_complete_streaming_response_legacy,
@ -254,6 +256,26 @@ class AnthropicPassthroughLoggingHandler:
),
)
@staticmethod
async def log_error_frame_as_failure(
attempt_logging_obj: LiteLLMLoggingObj,
request_body: Mapping[str, object],
all_chunks: Sequence[str | bytes],
error_event: tuple[str, str, int],
) -> None:
"""A stream closed at the provider's `event: error` frame is one failed attempt of the request: it logs
through the attempt-scoped copy its caller took at the close, so the usage it consumed never reaches
the success log of the attempt the router opens in its place."""
error_type, message, status_code = error_event
await asyncify(AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure)(
litellm_logging_obj=attempt_logging_obj, request_body=request_body, all_chunks=all_chunks
)
await attempt_logging_obj.dispatch_failure_handlers(
anthropic_error_frame_exception(error_type, message, status_code, str(attempt_logging_obj.model)),
"",
prefer_async_handlers=True,
)
@staticmethod
def _cost_partial_stream_or_zero(
partial_response: ModelResponse | TextCompletionResponse, model: str, logging_obj: LiteLLMLoggingObj

View file

@ -165,6 +165,7 @@ class PassThroughStreamingHandler:
)
logging_scheduled = False
stream_ended = False # rebind-ok: flipped once the upstream ran to its end, so the finally can tell a close-at-frame from a completed stream
model_name: Final = PassThroughStreamingHandler._extract_model_for_cost_injection(
request_body=request_body,
url_route=url_route,
@ -224,6 +225,7 @@ class PassThroughStreamingHandler:
)
if pending:
yield pending
stream_ended = True # rebind-ok: the finally reads it to spot a close-at-frame
# Stream completed cleanly. When the proxy armed deferred
# dispatch (post-call guardrails active), park the logging
# coroutine on logging_obj instead of enqueueing now, so
@ -267,13 +269,46 @@ class PassThroughStreamingHandler:
# a success would double-log the same request.
if not logging_scheduled and raw_bytes and response.status_code < 400:
logging_scheduled = True
superseded_error_event: Final = (
None
if stream_ended
else PassThroughStreamingHandler._error_frame_closed_at(endpoint_type, raw_bytes)
)
try:
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=_build_logging_coroutine())
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
async_coroutine=(
_build_logging_coroutine()
if superseded_error_event is None
else AnthropicPassthroughLoggingHandler.log_error_frame_as_failure(
attempt_logging_obj=litellm_logging_obj.attempt_scoped_copy(),
request_body=resolved_request_body,
all_chunks=tuple(raw_bytes),
error_event=superseded_error_event,
)
)
)
except Exception as e:
verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e)
else:
bind_budget_reservation_to_callbacks(litellm_logging_obj.litellm_params)
@staticmethod
def _error_frame_closed_at(endpoint_type: EndpointType, raw_bytes: Sequence[bytes]) -> tuple[str, str, int] | None:
"""The provider `event: error` frame of a stream the router closed because a retry or fallback took the
request over. That attempt is logged as its own failure, on a copy taken now so the retry's rebinding of
the shared logging object never reaches it. A stream that ran to its end or that the client closed keeps
logging through the success route, so its usage is still billed."""
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
closing_superseded_attempt,
parse_anthropic_error_event,
)
if endpoint_type != EndpointType.ANTHROPIC or not closing_superseded_attempt():
return None
complete_frames, _ = split_complete_sse_frames(b"".join(raw_bytes))
return parse_anthropic_error_event(complete_frames)
@staticmethod
async def route_streaming_logging_to_handler(
litellm_logging_obj: LiteLLMLoggingObj,

View file

@ -34,6 +34,7 @@ from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.litellm_core_utils.tool_fields import first_validation_error
from litellm.llms.openai.responses.guardrail_translation.handler import (
OpenAIResponsesHandler,
build_blocked_response,
@ -45,7 +46,12 @@ from litellm.proxy.auth.user_api_key_auth import (
user_api_key_auth,
user_api_key_auth_websocket,
)
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing, cost_headers, create_response
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
cost_headers,
create_response,
resolve_counted_tool_fields,
)
from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports
_read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
_safe_set_request_parsed_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
@ -57,6 +63,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
request_tags_from_request_data,
)
from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
from litellm.router import Router
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import (
REASONING_EFFORT,
@ -69,7 +76,6 @@ from litellm.types.utils import TokenCountResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy.utils import ProxyLogging
from litellm.router import Router
router: Final = APIRouter()
_RESPONSES_WS_CONFIG_VALUE_ADAPTER: Final[TypeAdapter[object | None]] = TypeAdapter(object | None)
@ -1376,6 +1382,15 @@ def _proxy_token_counter() -> _TokenCounter:
_token_counter_dep: Final = Depends(_proxy_token_counter)
def _proxy_llm_router() -> Router | None:
from litellm.proxy.proxy_server import llm_router
return llm_router
_llm_router_dep: Final = Depends(_proxy_llm_router)
def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse:
body: Final[_ResponsesApiErrorBody] = {
"error": {
@ -1432,6 +1447,8 @@ def _responses_input_as_token_count_messages(
async def responses_input_tokens(
request: Request,
token_counter: _TokenCounter = _token_counter_dep,
llm_router: Router | None = _llm_router_dep,
user_api_key_dict: UserAPIKeyAuth = _user_api_key_auth_dep,
):
"""
Count the input tokens of a Responses API request without calling the model.
@ -1464,6 +1481,13 @@ async def responses_input_tokens(
code="missing_required_parameter",
)
counted_fields: Final = {
name: data[name] for name in ("model", "tools", "drop_params", "additional_drop_params") if name in data
}
token_fields: Final = resolve_counted_tool_fields(counted_fields, llm_router, user_api_key_dict.team_id)
if isinstance(token_fields, ProxyException):
return _responses_invalid_request_response(message=token_fields.message, param=token_fields.param, code=None)
try:
payload: Final[_TokenCountPayload] = {
"model": model_name,
@ -1471,9 +1495,13 @@ async def responses_input_tokens(
input_value=input_value,
instructions=data.get("instructions"),
),
"tools": data.get("tools"),
"tools": token_fields.get("tools"),
}
token_request: Final = TokenCountRequest.model_validate(payload)
except ValidationError as e:
return _responses_invalid_request_response(
message=first_validation_error(e), param=str(e.errors(include_url=False)[0]["loc"][0]), code=None
)
except Exception as e:
return _responses_invalid_request_response(
message=f"Invalid request for token counting: {e}", param=None, code=None

View file

@ -3493,6 +3493,7 @@ class ProxyLogging:
Is triggered when self._is_proxy_only_error() returns True
"""
litellm_logging_obj: Logging | None = request_data.get("litellm_logging_obj", None)
set_up_by_request: Final = litellm_logging_obj is not None
if litellm_logging_obj is None:
from litellm._uuid import uuid
@ -3557,7 +3558,9 @@ class ProxyLogging:
input = request_data["input"]
litellm_logging_obj.model_call_details["input"] = input
body_shape_call_type = CallTypes.aembedding.value
resolved_call_type: Final = _call_type_for_route(route) or body_shape_call_type
resolved_call_type: Final = _call_type_for_route(route) or (
None if set_up_by_request else body_shape_call_type
)
if resolved_call_type is not None and litellm_logging_obj.call_type != CallTypes.pass_through.value:
litellm_logging_obj.call_type = resolved_call_type
litellm_logging_obj.model_call_details["call_type"] = resolved_call_type

View file

@ -123,6 +123,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
mask_sensitive_structure,
)
from litellm.litellm_core_utils.token_counter import offload_token_count
from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, anthropic_error_frame_exception
from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
from litellm.llms.base_llm.vector_store.transformation import (
@ -537,6 +538,7 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream])
_NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({})
_NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
@ -573,6 +575,21 @@ def _deployment_that_just_failed(request_metadata: object) -> str | None:
return model_id if isinstance(model_id, str) else None
def _drops_malformed_request_tools(
deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object]
) -> bool:
if isinstance(request.get("tools") or [], (list, tuple)) or not deployment_params.get("tools"):
return False
set_defaults: Final = {name: value for name, value in defaults.items() if value is not None}
return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools")
def _merged_tools(deployment_tools: Sequence[object], request_tools: object) -> list[object] | None:
if not isinstance(request_tools, (list, tuple)):
return None
return [*deployment_tools, *request_tools] if deployment_tools or request_tools else None
def _deployment_pick_attributes(model: str, request_kwargs: Mapping[str, object] | None) -> Mapping[str, str | int]:
"""Bounded attributes for one deployment pick; attempt is 1-based within the current model group."""
kwargs: Final = request_kwargs or {}
@ -4507,7 +4524,9 @@ class Router:
return deployment_pydantic_obj
@staticmethod
def _merge_tools_from_deployment(deployment: dict, kwargs: dict) -> None:
def _merge_tools_from_deployment(
deployment: dict, kwargs: dict, defaults: Mapping[str, object] = _NO_DEFAULT_PARAMS
) -> None:
"""
Merge tools from deployment litellm_params with request kwargs.
When both have tools, concatenate them (deployment tools first, then request tools).
@ -4518,11 +4537,12 @@ class Router:
dep_params = dep_params_raw
else:
dep_params = dep_params_raw.model_dump(exclude_none=True)
dep_tools: Final = dep_params.get("tools") or []
req_tools: Final = kwargs.get("tools") or []
if dep_tools or req_tools:
merged: Final = list(dep_tools) + list(req_tools)
kwargs["tools"] = merged
if _drops_malformed_request_tools(dep_params, defaults, kwargs):
for name in TOOL_FIELDS:
kwargs.pop(name, None)
merged_tools: Final = _merged_tools(dep_params.get("tools") or [], kwargs.get("tools") or [])
if merged_tools is not None:
kwargs["tools"] = merged_tools
if "tool_choice" not in kwargs and dep_params.get("tool_choice") is not None:
kwargs["tool_choice"] = dep_params["tool_choice"]
@ -4543,7 +4563,7 @@ class Router:
deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ())
):
kwargs.pop(key, None)
self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs)
self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=self.default_litellm_params)
model_info = deployment.get("model_info", {}).copy()
deployment_litellm_model_name = deployment["litellm_params"]["model"]
@ -6035,6 +6055,7 @@ class Router:
"""
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
aclose_if_supported,
aclose_superseded_attempt,
parse_anthropic_error_event,
parse_anthropic_refusal_stop_details,
)
@ -6044,6 +6065,14 @@ class Router:
source_iterator: Final = response
async def close_source() -> None:
with anyio.CancelScope(shield=True), contextlib.suppress(BaseException):
await aclose_if_supported(source_iterator)
async def close_superseded() -> None:
with anyio.CancelScope(shield=True), contextlib.suppress(BaseException):
await aclose_superseded_attempt(source_iterator)
async def stream_with_fallbacks() -> AsyncGenerator[bytes, None]:
from litellm.exceptions import MidStreamFallbackError
@ -6130,11 +6159,11 @@ class Router:
model,
initial_kwargs,
wrapper,
close_superseded,
):
yield item
finally:
with anyio.CancelScope(shield=True), contextlib.suppress(BaseException):
await aclose_if_supported(source_iterator)
await close_source()
# Referenced by stream_with_fallbacks via closure - assigned here, before
# the generator body ever runs, so the reference resolves fine despite
@ -6150,6 +6179,7 @@ class Router:
model: str,
initial_kwargs: dict[str, Any], # mutable-ok: handed to _aanthropic_messages_fallback_attempt, which mutates it
wrapper: "FallbackAwareAnthropicMessagesStream",
close_superseded: Callable[[], Awaitable[None]],
) -> AsyncGenerator[bytes, None]:
"""Turns a source-iterator failure into a fallback attempt or the error reaching the caller."""
from litellm.exceptions import MidStreamFallbackError
@ -6169,6 +6199,7 @@ class Router:
)
if fallback_error is None:
raise stream_error
await close_superseded()
outcome: Final = await self._astream_retry_same_group(
fallback_error,
initial_kwargs,

View file

@ -364,10 +364,10 @@ def filter_web_search_deployments(
if isinstance(healthy_deployments, dict):
return healthy_deployments
tools_value: Final[object] = request_kwargs.get("tools") or []
tools: Final = _OBJECT_TUPLE_ADAPTER.validate_python(tools_value)
is_web_search_request: Final = any(_is_web_search_tool(tool) for tool in tools)
tools: Final[object] = request_kwargs.get("tools")
is_web_search_request: Final = isinstance(tools, (list, tuple)) and any(
_is_web_search_tool(tool) for tool in _OBJECT_TUPLE_ADAPTER.validate_python(tools)
)
if not is_web_search_request:
return healthy_deployments

View file

@ -329,6 +329,12 @@ def _should_run_cooldown_logic(
)
return False
if isinstance(original_exception, litellm.MalformedRequestFieldError):
verbose_router_logger.debug(
"Should Not Run Cooldown Logic: litellm rejected the request before calling the deployment"
)
return False
if request_kwargs is not None and _is_per_user_provider_request(request_kwargs):
verbose_router_logger.debug(
"Should Not Run Cooldown Logic: the failure came from one caller's per-user provider session"

View file

@ -1195,6 +1195,16 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up
description="Maximum decisions calls in flight at once on the guardrail in each proxy worker. Implemented by guardrail='decision_model'.",
)
forward_messages_as_sent: bool | None = Field(
default=False,
description=(
"Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the "
"guardrail exactly as the client sent it, so rows can carry provider-native content blocks and "
"keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with "
"undeclared keys removed."
),
)
extra_headers: list[str] | None = Field(
default=None,
description=(

View file

@ -1,7 +1,7 @@
from collections.abc import Mapping, Sequence
from typing import Any, Final, Literal, cast # noqa: TID251 # JSON chat rows have no typed constructor across roles
from pydantic import ConfigDict, Field
from pydantic import ConfigDict, Field, SerializerFunctionWrapHandler, model_serializer
from typing_extensions import TypedDict
from litellm.types.llms.base import LiteLLMBaseModel
@ -17,12 +17,20 @@ class GuardrailToolParam(LiteLLMBaseModel):
"""A tool forwarded verbatim to the guardrail for inspection.
Built-in tools (code_interpreter, file_search, ...) have no ``function`` block
and stash their config in tool-specific keys, so only ``type`` is required and
``extra="allow"`` preserves the rest instead of stripping it.
and stash their config in tool-specific keys, and provider-native tools such as
Gemini ``function_declarations`` carry no ``type`` at all, so nothing is required
and ``extra="allow"`` preserves the rest instead of stripping it.
"""
model_config = ConfigDict(extra="allow")
type: str
type: str | None = None
@model_serializer(mode="wrap")
def _without_absent_type(self, handler: SerializerFunctionWrapHandler) -> Mapping[str, object]:
dumped: Final = cast("Mapping[str, object]", handler(self)) # cast-ok: handler dumps this model's fields
if self.type is not None:
return dumped
return {key: value for key, value in dumped.items() if key != "type"}
class GenericGuardrailAPIMetadata(TypedDict, total=False):
@ -62,6 +70,15 @@ class GenericGuardrailAPIOptionalParams(LiteLLMBaseModel):
),
)
forward_messages_as_sent: bool | None = Field(
default=False,
description=(
"When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can "
"carry provider-native content blocks and keys the OpenAI chat types do not declare. False "
"(default) sends OpenAI-format chat rows with undeclared keys removed."
),
)
streaming_end_of_stream_only: bool | None = Field(
default=None,
description=(

View file

@ -92,6 +92,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
from litellm.litellm_core_utils.tokenizer import strip_special_tokens
from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields
from litellm.rust_bridge import tokenizer as tokenizer_dispatch
from litellm.rust_bridge.catalog import decision
from litellm.rust_bridge.configuration import Decision
@ -474,6 +475,7 @@ from .exceptions import (
BudgetExceededError,
ContentPolicyViolationError,
ContextWindowExceededError,
MalformedRequestFieldError,
ModelNotMappedError,
NotFoundError,
OpenAIError,
@ -1495,7 +1497,23 @@ def check_coroutine(value) -> bool:
return get_coroutine_checker().is_async_callable(value)
async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str):
def _deployment_provider(kwargs: Mapping[str, object]) -> str:
model: Final = kwargs.get("model")
custom_llm_provider: Final = kwargs.get("custom_llm_provider")
api_base: Final = kwargs.get("api_base")
if not isinstance(model, str):
return ""
try:
return litellm.get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
)[1]
except BadRequestError:
return ""
async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str) -> dict[str, Any]:
"""
Allow modifying the request just before it's sent to the deployment.
@ -1506,7 +1524,16 @@ async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str)
except ValueError:
typed_call_type = None # unknown call type
modified_kwargs = kwargs.copy()
route: Final[object] = getattr(kwargs.get("litellm_logging_obj"), "call_type", None)
resolved: Final = resolve_deployment_tool_fields(kwargs, route)
if isinstance(resolved, MalformedField):
raise MalformedRequestFieldError(
message=resolved.message,
model=kwargs.get("model"),
llm_provider=_deployment_provider(kwargs),
body={"type": "invalid_request_error", "param": resolved.field},
)
modified_kwargs = dict(resolved)
custom_logger_class: Final = _get_cached_custom_logger()
for callback in litellm.callbacks:

View file

@ -1,6 +1,6 @@
# litellm/proxy/vector_stores/vector_store_registry.py
import json
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from itertools import chain
from typing import (
@ -11,7 +11,7 @@ from typing import (
get_args,
)
from pydantic import ConfigDict, TypeAdapter
from pydantic import ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import remove_items_at_indices
@ -34,6 +34,26 @@ else:
PrismaClient = Any
_DB_ROW: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
_TOOL: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True))
_STORE_ID_ITEMS: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
def _named_store_ids(store_ids: object) -> tuple[str, ...]:
if isinstance(store_ids, str):
return (store_ids,)
try:
items: Final = _STORE_ID_ITEMS.validate_python(store_ids)
except ValidationError:
return ()
return tuple(store_id for store_id in items if isinstance(store_id, str))
def _tool_vector_store_ids(tool: object) -> tuple[str, ...]:
try:
return _named_store_ids(_TOOL.validate_python(tool).get("vector_store_ids"))
except ValidationError:
return ()
class VectorStoreIndexRegistry:
@ -131,7 +151,7 @@ class VectorStoreRegistry:
return VectorStoreToolParams(**kwargs)
def get_vector_store_ids_to_run(self, non_default_params: dict, tools: list[dict] | None = None) -> list[str]:
def get_vector_store_ids_to_run(self, non_default_params: dict, tools: object = None) -> list[str]:
"""
Returns the vector store ids to run
@ -175,7 +195,7 @@ class VectorStoreRegistry:
tools_to_remove: Final[list[int]] = []
for i, tool in enumerate(tools):
tool_vector_store_ids = tool.get("vector_store_ids", [])
tool_vector_store_ids = _tool_vector_store_ids(tool)
if not tool_vector_store_ids:
continue
@ -395,14 +415,15 @@ class VectorStoreRegistry:
return vector_stores_to_run
def _get_vector_store_ids_from_tool_calls(self, tools: list[dict] | None, vector_store_ids: list[str]) -> list[str]:
def _get_vector_store_ids_from_tool_calls(self, tools: object, vector_store_ids: list[str]) -> list[str]:
"""
Returns the vector store ids from the tool calls
"""
return [
*vector_store_ids,
*chain.from_iterable(tool["vector_store_ids"] for tool in tools or () if "vector_store_ids" in tool),
]
try:
validated_tools: Final = _ITEMS.validate_python(tools)
except ValidationError:
return vector_store_ids
return [*vector_store_ids, *chain.from_iterable(_tool_vector_store_ids(tool) for tool in validated_tools)]
def load_vector_stores_from_config(self, vector_stores_config: list[dict]):
"""

View file

@ -112,6 +112,8 @@ ignored_function_names = [
"_get_wildcard_deployments", # Tested through the get_model_list_of_routed_group wildcard test in test_router.py
"_is_fallback_hop", # Tested through the order fallback hop tests in test_router_order_fallback.py
"_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py
"_merged_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
"_drops_malformed_request_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
"_collect_ptu_share_teams", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
"_resolve_team_ptu_ceiling", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
"_register_continuation_prefill_check", # Tested through the continuation tests in test_router.py

View file

@ -25,3 +25,15 @@ By the domain a user would name: `pricing`, `spend`, `routing`, `mcp`. A file on
directory that a `GROUPS` entry in `run.py` selects. There is no manifest and no `covers` marker on new
tests. A product bug the test exposes is `pytest.skip("BUG: <symptom>")` at the top of the body, not a
fix in the test and not a deletion. Needs no network, proxy, DB or Redis: `tests/unit`
CircleCI's integration groups run every file here except the OCI and sandbox tests, which no CI runs, and
the files in `GITHUB_FILES` in `run.py`. CircleCI runs on main, not on pull requests, so a new file here
first runs after its PR merges. Wanting a file to run before merge is no reason to change CI, and a new
dedicated workflow for it is the same mistake as a shared shard. Write the file, leave `GITHUB_FILES` and
every workflow alone, and tell the user it first runs on main. `GITHUB_FILES` is only for a file that
needs what CircleCI's groups cannot give it, like the images `image-scan.yml` builds or the plain pytest
run CircleCI's `roi-database` job gives `test_roi_observed.py`. Add a file there only in that case,
together with its dedicated job, and list a new dedicated GitHub workflow with the directory it owns in
`DEDICATED_INTEGRATION_WORKFLOWS` in `.github/scripts/assert_ci_coverage.py`. The `ci-coverage` check
fails a file here that a GitHub workflow runs unless it is in `GITHUB_FILES` and that workflow owns its
directory (#46780)

View file

@ -4,7 +4,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local
The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport
The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so they run in the GitHub Actions `redis-compat` matrix against every supported redis-py major rather than in `run_integration.sh`
The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so CircleCI's `sdk` group installs `redis-server` before it runs them
The `cost` group is driven by the `CostTrackingTestCase` literals under `cost_calculation/cases/<provider>.py` (gathered by `cost_calculation/catalog.py`), with the scripted cost map in `cost_calculation/cost_map.py`. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/<scenario_id>`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL`
@ -26,7 +26,7 @@ Reuse the existing canned provider handlers through `_support/upstream.py`. It r
Owned proxies disable `.env` loading by default. A contract testing dotenv behavior can explicitly set `PYTHON_DOTENV_DISABLED=0` in its owned environment. The runner drops inherited `*_API_KEY` variables, so a contract that needs one sets a synthetic value in its owned environment. An explicit scratch writer database drops an inherited reader with a different database name; pass both database URLs when the contract needs a paired reader. Database relays close listeners, active connections and their event loops on teardown. Upstream startup failures reap the child process before returning the failure. Custom subprocesses, direct SQL and background tasks still need an explicit owner in the test
Owned proxy workers configure a 32-thread executor before application startup, preserving additional worker startup hooks. Held synchronous provider calls must not consume a CPU-dependent executor limit before the rest of a fault burst can start. The proxy entrypoint imports LiteLLM inside `main`, so spawned workers begin supervision before importing the application. Worker healthchecks have a five-second deadline by default, so uvicorn kills a worker that stays paused longer than that. Tests never signal proxy processes themselves. `OwnedProxy.kill_worker` SIGKILLs a worker and returns once its replacement answers readiness with its own pid, `replacing_worker` does the same for a kill that has to land mid-burst, and `suspended` pauses a worker or the supervisor, which only works on a proxy launched with `healthcheck=WorkerHealthcheck.LENIENT`. Helper processes that are not the proxy stop through `stop_child`, `stop_processes` and `stop_process_group` and pause through `paused`. `scripts/check_test_quality.py` rule TQ010 rejects a raw `os.kill`, SIGKILL / SIGSTOP / SIGCONT, `.kill()`, `.terminate()`, `.suspend()` or `.send_signal()` anywhere under `tests/integration` outside `_support`, and TQ011 rejects an assertion that caps an elapsed time with a fixed number. Bound timings with `_support/timing.py` instead, relative to a measured control or to the injected delay. `run.py` measures how long this box takes to import the proxy application before it starts pytest, records it in `boot-calibration.json` beside the results and exports it as `INTEGRATION_IMPORT_SECONDS`. Readiness deadlines, graceful stops and the default per-test timeout are multiples of that measurement (`_support/budget.py`), so a slower box gets longer budgets and a two-worker proxy gets twice the readiness deadline of a one-worker one. Readiness requires responsive worker identities that are still alive and a positively identified owned listener. Unreadable socket records cannot establish ownership. Crash tests own their client tasks with `TaskGroup`, release held peers in `finally` and verify that the intended worker dies. A test requiring traffic on both workers prepares that traffic explicitly instead of assuming OS load balancing, for example by sending each half while the other worker is suspended
Owned proxy workers configure a 32-thread executor before application startup, preserving additional worker startup hooks. Held synchronous provider calls must not consume a CPU-dependent executor limit before the rest of a fault burst can start. The proxy entrypoint imports LiteLLM inside `main`, so spawned workers begin supervision before importing the application. Worker healthchecks have a five-second deadline by default, so uvicorn kills a worker that stays paused longer than that. Tests never signal proxy processes themselves. `OwnedProxy.kill_worker` SIGKILLs a worker and returns once its replacement answers readiness with its own pid, `replacing_worker` does the same for a kill that has to land mid-burst, and `suspended` pauses a worker or the supervisor, which only works on a proxy launched with `healthcheck=WorkerHealthcheck.LENIENT`. The `kill()` that `replacing_worker` yields returns as soon as the worker is dead, so a test can release requests it holds on the surviving worker right away. Uvicorn may take a full worker healthcheck to notice the death, so the block waits for the supervisor to reap the worker and for the replacement only when it exits. Helper processes that are not the proxy stop through `stop_child`, `stop_processes` and `stop_process_group` and pause through `paused`. `scripts/check_test_quality.py` rule TQ010 rejects a raw `os.kill`, SIGKILL / SIGSTOP / SIGCONT, `.kill()`, `.terminate()`, `.suspend()` or `.send_signal()` anywhere under `tests/integration` outside `_support`, and TQ011 rejects an assertion that caps an elapsed time with a fixed number. Bound timings with `_support/timing.py` instead, relative to a measured control or to the injected delay. `run.py` measures how long this box takes to import the proxy application before it starts pytest, records it in `boot-calibration.json` beside the results and exports it as `INTEGRATION_IMPORT_SECONDS`. Readiness deadlines, graceful stops and the default per-test timeout are multiples of that measurement (`_support/budget.py`), so a slower box gets longer budgets and a two-worker proxy gets twice the readiness deadline of a one-worker one. Readiness requires responsive worker identities that are still alive and a positively identified owned listener. Unreadable socket records cannot establish ownership. Crash tests own their client tasks with `TaskGroup`, release held peers in `finally` and verify that the intended worker dies. A test requiring traffic on both workers prepares that traffic explicitly instead of assuming OS load balancing, for example by sending each half while the other worker is suspended
Any port the harness picks before the server that listens on it has started comes from `allocate_port()` in `_support/ports.py`. That covers owned proxies, scripted upstreams, fake providers, OTLP sinks, database relays and Redis. A server that binds port 0 itself and keeps that socket, such as `wire_server` without a port, needs no allocation. Each pytest process claims a block of 200 ports between 10000 and 32767 by holding an exclusive lock file per block in the system temp directory, so two runs on one machine, such as parallel worktrees, never share a block, and the OS drops the lock when the process exits. The range sits above the fixed service ports and below the Linux and macOS ephemeral ranges, so a port-0 bind or an outbound connection never lands in a block. Ports inside a block go out in order and are never handed out twice in a run, a port that something outside the harness already holds is skipped, and a process that uses up its block claims the next free one. A test that restarts a server on the same port allocates that port up front instead of reading it back from the first server. TQ012 in `scripts/check_test_quality.py` rejects reading a port back from a port-0 bind anywhere under `tests/integration`

View file

@ -150,9 +150,9 @@ def _is_reaped(process: psutil.Process) -> bool:
return False
def _wait_until_reaped(process: psutil.Process, supervisor: psutil.Process, seconds: float) -> None:
def _wait_until_reaped(process: psutil.Process, seconds: float) -> None:
deadline: Final = time.monotonic() + seconds
while not _is_reaped(process) and not _is_stopped(supervisor):
while not _is_reaped(process):
assert time.monotonic() < deadline, f"The supervisor did not reap worker {process.pid} within {seconds:.0f}s"
time.sleep(0.05)
@ -160,8 +160,6 @@ def _wait_until_reaped(process: psutil.Process, supervisor: psutil.Process, seco
@dataclass(frozen=True, slots=True)
class WorkerKill:
victim: psutil.Process
supervisor: psutil.Process
reap_seconds: float
fired: threading.Event
def freeze(self) -> psutil.Process:
@ -173,7 +171,6 @@ class WorkerKill:
with suppress(psutil.NoSuchProcess):
self.victim.kill()
_wait_until_gone(self.victim, STRICT_HEALTHCHECK_SECONDS)
_wait_until_reaped(self.victim, self.supervisor, self.reap_seconds)
self.fired.set()
@ -250,22 +247,22 @@ class OwnedProxy:
assert pid in before, f"{pid} is not a serving worker of this proxy: {sorted(before)}"
return psutil.Process(pid), before
def _worker_kill(self, victim: psutil.Process) -> WorkerKill:
reap_seconds: Final = healthcheck_seconds(self.healthcheck) + STRICT_HEALTHCHECK_SECONDS
return WorkerKill(victim, psutil.Process(self.process.pid), reap_seconds, threading.Event())
def _replacement_of(self, victim: psutil.Process, before: frozenset[int]) -> int:
_wait_until_reaped(victim, healthcheck_seconds(self.healthcheck) + STRICT_HEALTHCHECK_SECONDS)
return self._await_replacement(before)
@contextmanager
def replacing_worker(self, pid: int) -> Iterator[WorkerKill]:
victim, before = self._victim(pid)
kill: Final = self._worker_kill(victim)
kill: Final = WorkerKill(victim, threading.Event())
yield kill
assert kill.fired.is_set(), f"replacing_worker({pid}) exited without killing the worker"
self._await_replacement(before)
self._replacement_of(victim, before)
def kill_worker(self, pid: int) -> int:
victim, before = self._victim(pid)
self._worker_kill(victim)()
return self._await_replacement(before)
WorkerKill(victim, threading.Event())()
return self._replacement_of(victim, before)
@contextmanager
def suspended(self, pid: int) -> Iterator[psutil.Process]:

View file

@ -53,6 +53,13 @@ SELECT encode(sha256(convert_to(%s || '-' || n::text, 'UTF8')), 'hex'), %s, '{}'
FROM generate_series(1, %s::int) AS n
"""
TAKE_TEAM_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext(%s))"
LOCK_MODEL_ROW_SQL: Final = 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s FOR UPDATE'
LOCK_HELD_PAST_DEFAULT_TRANSACTION_SECONDS: Final = 7
OLDEST_TRANSACTION_BLOCKED_BY_SQL: Final = """
SELECT coalesce(max(extract(epoch FROM clock_timestamp() - xact_start)), 0)::float AS age
FROM pg_stat_activity
WHERE %s::int = ANY(pg_blocking_pids(pid))
"""
# Sessions blocked on the advisory lock the given backend holds, and nothing else on the shared rig.
WAITERS_ON_HELD_LOCK_SQL: Final = """
SELECT count(*)::int AS waiting
@ -131,6 +138,13 @@ def _waiters_on_lock_held_by(backend_pid: int) -> int:
return waiting
def _oldest_transaction_blocked_by(backend_pid: int) -> float:
rows: Final = read_rows(OLDEST_TRANSACTION_BLOCKED_BY_SQL, (str(backend_pid),))
age: Final = rows[0]["age"]
assert isinstance(age, float)
return age
def _remove_team_by_sql(team_id: str) -> None:
"""Cleanup for a team the test expects to have deleted itself. Whatever a failed delete left behind
(row, memberships, `teams` references) goes by SQL so the shared rig stays clean without sending
@ -161,6 +175,38 @@ def _create_team(scenario: Scenario, **fields: JsonValue) -> str:
return team_id
def _create_team_model(scenario: Scenario, team_id: str) -> str:
created: Final = scenario.gateway.post(
"/model/new",
{
"model_name": f"integration-{uuid.uuid4().hex}",
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-integration"},
"model_info": {"team_id": team_id},
},
)
model_id: Final = string_value(object_value(created["model_info"])["id"])
scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (model_id,))
return model_id
def _team_model_rows(model_id: str) -> list[dict[str, JsonValue]]:
return read_rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (model_id,))
def _delete_while_locked(gateway: Gateway, team_id: str, lock_sql: str, lock_target: str) -> httpx.Response:
with ThreadPoolExecutor(max_workers=1) as pool:
with psycopg.connect(os.environ["DATABASE_URL"]) as holder:
holder.execute(lock_sql, (lock_target,))
pending: Final = _post_in_background(pool, gateway, "/team/delete", {"team_ids": [team_id]})
eventually(
lambda: _oldest_transaction_blocked_by(holder.info.backend_pid),
lambda age: age > LOCK_HELD_PAST_DEFAULT_TRANSACTION_SECONDS,
seconds=30,
)
assert not pending.done(), "delete returned while the lock was still held"
return pending.result(timeout=60)
def _generate_key(scenario: Scenario, **fields: JsonValue) -> str:
"""A key the team delete is expected to remove; cleanup only deletes it if it is still live."""
key: Final = string_value(scenario.gateway.post("/key/generate", fields)["key"])
@ -392,6 +438,36 @@ def test_delete_waits_for_the_team_advisory_lock_and_completes_after_release(
assert len(_tombstones(team)) == 1, _tombstones(team)
def test_delete_completes_after_waiting_on_a_locked_team_model_row_past_the_default_transaction_timeout(
gateway: Gateway,
) -> None:
with gateway.scenario() as scenario:
team: Final = _create_team(scenario)
model_id: Final = _create_team_model(scenario, team)
response: Final = _delete_while_locked(gateway, team, LOCK_MODEL_ROW_SQL, model_id)
assert response.status_code == 200, response.text
assert response.json() == {"deleted_teams": [team]}
assert _team_rows(team) == []
assert _team_model_rows(model_id) == []
assert len(_tombstones(team)) == 1, _tombstones(team)
def test_delete_completes_after_waiting_on_the_team_lock_past_the_default_transaction_timeout(
gateway: Gateway,
) -> None:
with gateway.scenario() as scenario:
user: Final = scenario.user()
team: Final = _create_team(scenario)
_bulk_member_add(gateway, team, [user])
response: Final = _delete_while_locked(gateway, team, TAKE_TEAM_LOCK_SQL, team)
assert response.status_code == 200, response.text
assert response.json() == {"deleted_teams": [team]}
assert _team_rows(team) == []
assert _membership_user_ids(team) == []
assert _user_teams(user) == []
assert len(_tombstones(team)) == 1, _tombstones(team)
def test_deleting_two_teams_sharing_a_member_in_one_call_clears_both_from_the_member(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
user: Final = scenario.user()

View file

@ -40,6 +40,7 @@ FirstAttempt = Literal[
"drop_after_message_start",
"error_frame",
"error_frame_after_message_start",
"error_frame_after_content",
"http_status",
"drop_after_content",
]
@ -66,6 +67,8 @@ class _Failure:
return status_reply(self.status)
case "drop_after_content":
return dropping_reply(chunks, abort_after=3)
case "error_frame_after_content":
return stream_reply((*chunks[:3], error_frame(self.status, f"scripted {self.status}")))
_MID_STREAM_FAILURES: Final = (
@ -124,12 +127,12 @@ def _stream(gateway: Gateway, body: Mapping[str, JsonValue]) -> tuple[int, tuple
return response.status_code, parse_sse(response.text)
def _rows(request_id: str) -> list[dict[str, JsonValue]]:
return read_rows('SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (request_id,))
def _success_rows(request_id: str) -> list[dict[str, JsonValue]]:
return eventually(
lambda: read_rows('SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (request_id,)),
lambda rows: len(rows) >= 1,
seconds=70,
)
return eventually(lambda: _rows(request_id), lambda rows: len(rows) >= 1, seconds=70)
def _assert_completed_by_retry(
@ -141,6 +144,8 @@ def _assert_completed_by_retry(
assert delta_text(events) == _TEXT, events
assert [request.target for request in wire.drain()] == ["/v1/messages"] * attempts
assert _success_rows(_served_id(prompt, attempts)) == [{"status": "success", "model_group": model}]
superseded_rows: Final = {attempt: _rows(_served_id(prompt, attempt)) for attempt in range(1, attempts)}
assert not any(superseded_rows.values()), superseded_rows
def _assert_rejected_before_content(
@ -302,6 +307,23 @@ def test_stream_dropping_after_content_is_not_retried(gateway: Gateway) -> None:
assert [request.target for request in wire.drain()] == ["/v1/messages"]
def test_stream_error_frame_after_content_is_not_retried_and_keeps_its_spend_row(gateway: Gateway) -> None:
prompt: Final = _prompt()
attempts: Final = Attempts()
with (
wire_server(_upstream(prompt, _Failure("error_frame_after_content", 529), attempts)) as wire,
gateway.scenario() as scenario,
):
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, num_retries=1)
status, events = _stream(gateway, _body(model, prompt, "deployment"))
assert status == 200, events
assert event_types(events) == (*LIFECYCLE[:3], "error"), events
assert message_id(events) == _served_id(prompt, 1), events
assert delta_text(events) == _TEXT, events
assert [request.target for request in wire.drain()] == ["/v1/messages"]
assert [row["model_group"] for row in _success_rows(_served_id(prompt, 1))] == [model]
def test_stream_rejected_with_401_before_it_opens_is_not_retried(gateway: Gateway) -> None:
prompt: Final = _prompt()
attempts: Final = Attempts()

View file

@ -364,7 +364,7 @@ def test_spend_log_keeps_the_tools_the_caller_sent(rig: Rig, model: str, guardra
assert object_value(rows[0]["proxy_server_request"])["tools"] == [strict_tool()]
@pytest.mark.parametrize("tools", [7, "x" * 5000, ["lint"]], ids=["int", "five_kb", "list_of_str"])
@pytest.mark.parametrize("tools", [7, "x" * 5000, ["lint"], ""], ids=["int", "five_kb", "list_of_str", "empty_string"])
def test_a_malformed_tools_field_is_answered_with_an_error_and_the_proxy_keeps_serving(
rig: Rig, tools: JsonValue
) -> None:
@ -379,14 +379,6 @@ def test_a_malformed_tools_field_is_answered_with_an_error_and_the_proxy_keeps_s
assert reply_text(control_status, control_raw, streaming=False) == f"linted {control_nonce.removeprefix('nonce-')}"
def test_an_empty_string_tools_field_reaches_claude_as_sent(rig: Rig) -> None:
request_nonce: Final = nonce()
status, raw = rig.send(messages_body(CLAUDE_MODEL, request_nonce, tools=""))
assert reply_text(status, raw, streaming=False) == f"linted {request_nonce.removeprefix('nonce-')}"
assert sent_body(rig.received(request_nonce), CLAUDE_TARGET)["tools"] == ""
@pytest.mark.parametrize(
("model", "target"),
[(CLAUDE_MODEL, CLAUDE_TARGET), (OPENAI_MODEL, RESPONSES_TARGET)],

View file

@ -0,0 +1,308 @@
import json
import threading
from collections.abc import Iterator, Mapping
from contextlib import ExitStack
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final
import pytest
import yaml
from integration._support.anthropic_sse import message_json
from integration._support.client import Gateway, eventually
from integration._support.openai_wire import chat_reply
from integration._support.ports import allocate_port
from integration._support.process import OwnedProxy, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
GEMINI_MODEL: Final = "gemini/gemini-3.8-flash"
ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-5-5"
OPENAI_MODEL: Final = "openai/gpt-5.6"
ECHO_GEMINI_TOOLS: Final = "echo-gemini-tools"
NONE_CHAT_ROWS: Final = "none-chat-rows"
NONE_CHAT_ROWS_DEFAULT: Final = "none-chat-rows-default"
NONE_ANTHROPIC_IMAGE: Final = "none-anthropic-image"
ECHO_ANTHROPIC_MESSAGES: Final = "echo-anthropic-messages"
FORWARDS_MESSAGES_AS_SENT: Final = frozenset({NONE_CHAT_ROWS, NONE_ANTHROPIC_IMAGE, ECHO_ANTHROPIC_MESSAGES})
FAIL_OPEN_CONNECT_ERROR: Final = "fail-open-connect-error"
FAIL_OPEN_TIMEOUT: Final = "fail-open-timeout"
FAIL_OPEN_HTTP_STATUS: Final = "fail-open-http-status"
FAIL_OPEN_LINE: Final = "Generic Guardrail API error (fail-open)"
PNG: Final = (
"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAFklEQVR4nGO4IydHEmIY1TCqYfhqAACaMxgQdGu1YAAAAABJRU5ErkJggg=="
)
_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
_JSON: Final = TypeAdapter(dict[str, JsonValue])
def _verdict_none(_request: Request) -> Reply:
return Reply(body=json.dumps({"action": "NONE"}).encode())
def _verdict_echo(request: Request) -> Reply:
body: Final = _JSON.validate_json(request.body)
echoed: Final = {key: body[key] for key in ("structured_messages", "tools") if body.get(key)}
return Reply(body=json.dumps({"action": "NONE", **echoed}).encode())
def _unavailable(_request: Request) -> Reply:
return Reply(status=503, body=b'{"error": "guardrail unavailable"}')
def _gemini_reply(_request: Request) -> Reply:
return Reply(
body=json.dumps(
{
"candidates": [{"content": {"role": "model", "parts": [{"text": "ok"}]}, "finishReason": "STOP"}],
"usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 1, "totalTokenCount": 4},
}
).encode()
)
def _anthropic_reply(_request: Request) -> Reply:
return Reply(body=message_json("msg_guardrail_wire", "claude-sonnet-5-5", "ok"))
def _openai_reply(_request: Request) -> Reply:
return chat_reply("chatcmpl-guardrail-wire", "gpt-5.6", "ok", stream=False)
def _guardrail(name: str, api_base: str, **params: JsonValue) -> dict[str, JsonValue]:
return {
"guardrail_name": name,
"litellm_params": {
"guardrail": "generic_guardrail_api",
"mode": "pre_call",
"default_on": False,
"api_base": api_base,
"api_key": "synthetic-guardrail-key",
**params,
},
}
@dataclass(frozen=True, slots=True)
class GuardrailRig:
owned: OwnedProxy
policies: Mapping[str, Wire]
@pytest.fixture(scope="module")
def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[GuardrailRig]:
directory: Final = tmp_path_factory.mktemp("generic-guardrail-wire")
release: Final = threading.Event()
def held_until_teardown(request: Request) -> Reply:
release.wait()
return _verdict_none(request)
with ExitStack() as stack:
policies: Final = MappingProxyType(
{
ECHO_GEMINI_TOOLS: stack.enter_context(wire_server(_verdict_echo)),
NONE_CHAT_ROWS: stack.enter_context(wire_server(_verdict_none)),
NONE_CHAT_ROWS_DEFAULT: stack.enter_context(wire_server(_verdict_none)),
NONE_ANTHROPIC_IMAGE: stack.enter_context(wire_server(_verdict_none)),
ECHO_ANTHROPIC_MESSAGES: stack.enter_context(wire_server(_verdict_echo)),
}
)
stalled: Final = stack.enter_context(wire_server(held_until_teardown))
stack.callback(release.set)
unavailable: Final = stack.enter_context(wire_server(_unavailable))
config: Final = _JSON.validate_python(yaml.safe_load(_CONFIG.read_text()))
path: Final = directory / "generic-guardrail-wire.yaml"
path.write_text(
yaml.safe_dump(
{
**config,
"guardrails": [
*(
_guardrail(name, wire.url, forward_messages_as_sent=name in FORWARDS_MESSAGES_AS_SENT)
for name, wire in policies.items()
),
_guardrail(FAIL_OPEN_CONNECT_ERROR, f"http://127.0.0.1:{allocate_port()}", fail_on_error=False),
_guardrail(FAIL_OPEN_TIMEOUT, stalled.url, fail_on_error=False, timeout=1),
_guardrail(FAIL_OPEN_HTTP_STATUS, unavailable.url, fail_on_error=False),
],
}
)
)
owned: Final = stack.enter_context(owned_proxy_process(session_gateway, directory, {}, config=path))
yield GuardrailRig(owned, policies)
def _bodies(wire: Wire) -> list[dict[str, JsonValue]]:
return [_JSON.validate_json(request.body) for request in wire.drain()]
def _fail_open_lines(log: Path, guardrail: str) -> list[str]:
return [
line
for line in log.read_text(errors="replace").splitlines()
if FAIL_OPEN_LINE in line and f"guardrail_name={guardrail} " in line
]
def test_gemini_native_tools_reach_the_guardrail_and_gemini_as_sent(rig: GuardrailRig) -> None:
tools: Final = [
{"function_declarations": [{"name": "get_weather", "parameters": {"type": "object", "properties": {}}}]},
{"googleSearch": {}},
]
with wire_server(_gemini_reply) as gemini, rig.owned.gateway.scenario() as scenario:
model: Final = scenario.model(model=GEMINI_MODEL, api_base=gemini.url, api_key="synthetic-gemini-key")
response: Final = rig.owned.gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": "weather in Paris?"}],
"tools": tools,
"guardrails": [ECHO_GEMINI_TOOLS],
},
)
assert response.status_code == 200, response.text
assert [body["tools"] for body in _bodies(rig.policies[ECHO_GEMINI_TOOLS])] == [tools]
sent: Final = _bodies(gemini)
assert len(sent) == 1, sent
assert {"googleSearch": {}} in sent[0]["tools"], sent
assert [tool for tool in sent[0]["tools"] if "type" in tool] == [], sent
CHAT_ROWS: Final = (
{"role": "user", "name": "pat", "content": "weather in Paris?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"index": 0, "id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
)
OPENAI_CHAT_ROWS: Final = (
{"role": "user", "content": "weather in Paris?"},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}],
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
)
@pytest.mark.parametrize(
("guardrail", "seen"),
[(NONE_CHAT_ROWS, CHAT_ROWS), (NONE_CHAT_ROWS_DEFAULT, OPENAI_CHAT_ROWS)],
ids=["forward_messages_as_sent", "default"],
)
def test_guardrail_sees_chat_rows_as_sent_only_when_configured(
rig: GuardrailRig, guardrail: str, seen: tuple[dict[str, JsonValue], ...]
) -> None:
messages: Final = list(CHAT_ROWS)
with wire_server(_openai_reply) as openai_peer, rig.owned.gateway.scenario() as scenario:
model: Final = scenario.model(
model=OPENAI_MODEL, api_base=f"{openai_peer.url}/v1", api_key="synthetic-openai-key"
)
response: Final = rig.owned.gateway.request(
"POST", "/v1/chat/completions", {"model": model, "messages": messages, "guardrails": [guardrail]}
)
assert response.status_code == 200, response.text
assert [body["structured_messages"] for body in _bodies(rig.policies[guardrail])] == [list(seen)]
assert len(openai_peer.drain()) == 1
def test_guardrail_sees_an_anthropic_image_block_on_chat_completions_as_sent(rig: GuardrailRig) -> None:
image: Final = [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG}}]
with wire_server(_anthropic_reply) as anthropic_peer, rig.owned.gateway.scenario() as scenario:
model: Final = scenario.model(
model=ANTHROPIC_MODEL, api_base=anthropic_peer.url, api_key="synthetic-anthropic-key"
)
rig.owned.gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"max_tokens": 16,
"messages": [{"role": "user", "content": image}, {"role": "user", "content": "what color?"}],
"guardrails": [NONE_ANTHROPIC_IMAGE],
},
)
seen: Final = _bodies(rig.policies[NONE_ANTHROPIC_IMAGE])
assert [body["structured_messages"][0]["content"] for body in seen] == [image], seen
def test_echoing_guardrail_leaves_the_anthropic_messages_request_unchanged(rig: GuardrailRig) -> None:
messages: Final = [
{
"role": "user",
"content": [
{"type": "text", "text": "What color is this square? Also check the weather in Paris."},
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG}},
{"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Trip notes."}},
],
},
{
"role": "assistant",
"content": [{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}}],
},
{
"role": "user",
"content": [
{"type": "tool_result", "tool_use_id": "toolu_1", "content": "sunny, 21C"},
{"type": "text", "text": "Answer both in one sentence."},
],
},
]
tools: Final = [
{
"name": "get_weather",
"description": "Weather for a city",
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
}
]
with wire_server(_anthropic_reply) as anthropic_peer, rig.owned.gateway.scenario() as scenario:
model: Final = scenario.model(
model=ANTHROPIC_MODEL, api_base=anthropic_peer.url, api_key="synthetic-anthropic-key"
)
response: Final = rig.owned.gateway.request(
"POST",
"/v1/messages",
{
"model": model,
"max_tokens": 64,
"messages": messages,
"tools": tools,
"guardrails": [ECHO_ANTHROPIC_MESSAGES],
},
)
assert response.status_code == 200, response.text
assert len(rig.policies[ECHO_ANTHROPIC_MESSAGES].drain()) == 1
assert [body["messages"] for body in _bodies(anthropic_peer)] == [messages]
@pytest.mark.parametrize(
("guardrail", "tokens"),
[
(FAIL_OPEN_CONNECT_ERROR, ("error_type=ConnectError",)),
(FAIL_OPEN_TIMEOUT, ("error_type=Timeout",)),
(FAIL_OPEN_HTTP_STATUS, ("error_type=HTTPStatusError", "http_status_code=503")),
],
ids=["connect_error", "timeout", "http_status"],
)
def test_fail_open_log_line_names_the_error_type(rig: GuardrailRig, guardrail: str, tokens: tuple[str, ...]) -> None:
with wire_server(_openai_reply) as openai_peer, rig.owned.gateway.scenario() as scenario:
model: Final = scenario.model(
model=OPENAI_MODEL, api_base=f"{openai_peer.url}/v1", api_key="synthetic-openai-key"
)
response: Final = rig.owned.gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": "hi"}], "guardrails": [guardrail]},
)
assert response.status_code == 200, response.text
assert len(openai_peer.drain()) == 1
lines: Final = eventually(lambda: _fail_open_lines(rig.owned.log, guardrail), bool, seconds=30)
assert [line for line in lines if not all(token in line for token in tokens)] == [], lines

View file

@ -1147,8 +1147,8 @@ def test_messages_count_tokens_surfaces_the_mantle_error_for_a_non_text_system(
@pytest.mark.parametrize(
"tools",
["", "x" * 5120, ["get_weather"]],
ids=["empty_string", "5kb_string", "list_of_strings"],
[5, "", "x" * 5120, ["get_weather"]],
ids=["int", "empty_string", "5kb_string", "list_of_strings"],
)
def test_messages_count_tokens_rejects_malformed_tools_without_calling_either_peer(
counting_proxy: OwnedProxy, mantle_port: int, tools: JsonValue
@ -1161,8 +1161,8 @@ def test_messages_count_tokens_rejects_malformed_tools_without_calling_either_pe
):
model: Final = _deployment(scenario, runtime.url)
refused: Final = _count(gateway, {**_bare(model), "tools": tools})
assert 400 <= refused.status_code < 600, refused.text
assert "input_tokens" not in refused.text, refused.text
assert refused.status_code == 400, refused.text
assert object_value(_JSON_OBJECT.validate_json(refused.content)["error"])["type"] == "invalid_request_error"
assert runtime.drain() == () and mantle.drain() == ()
assert _generated_then_counted(gateway.client, gateway.key, model, _bare(model)) == (200, 200, _MANTLE_COUNT)
assert _mantle_bodies(mantle) == (_MANTLE_BARE,)
@ -1185,8 +1185,13 @@ def test_messages_count_tokens_without_messages_is_rejected(
assert runtime.drain() == () and mantle.drain() == ()
@pytest.mark.parametrize(
"fields",
[{}, {"tools": 5}, {"web_search_options": None}],
ids=["well_formed", "int_tools", "null_web_search_options"],
)
def test_messages_count_tokens_unauthenticated_request_never_reaches_either_peer(
counting_proxy: OwnedProxy, mantle_port: int
counting_proxy: OwnedProxy, mantle_port: int, fields: dict[str, JsonValue]
) -> None:
gateway: Final = counting_proxy.gateway
with (
@ -1196,7 +1201,7 @@ def test_messages_count_tokens_unauthenticated_request_never_reaches_either_peer
):
model: Final = _deployment(scenario, runtime.url)
response: Final = gateway.request(
"POST", "/v1/messages/count_tokens", _full(model), key="sk-not-a-key-this-proxy-issued"
"POST", "/v1/messages/count_tokens", {**_full(model), **fields}, key="sk-not-a-key-this-proxy-issued"
)
assert response.status_code == 401, response.text
assert "input_tokens" not in response.text, response.text

View file

@ -0,0 +1,619 @@
from __future__ import annotations
import json
import os
import re
import threading
import uuid
from collections.abc import Iterator
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import ExitStack
from dataclasses import dataclass
from pathlib import Path
from queue import SimpleQueue
from typing import Final, Literal, TypeAlias
import httpx
import psutil
import pytest
from integration._support.anthropic_sse import error_body, message_json, message_stream
from integration._support.client import Gateway, eventually, gateway_from_environment
from integration._support.database import write_rows
from integration._support.openai_wire import chat_reply, responses_reply
from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from integration._support.worker_readiness import wait_for_worker_readiness
from pydantic import JsonValue, TypeAdapter
_OPENAI_BACKEND: Final = "gpt-4o-mini"
_CLAUDE_BACKEND: Final = "claude-haiku-4-5"
_GEMINI_BACKEND: Final = "gemini-2.5-flash"
_PROVIDER_KEY: Final = "scripted-malformed-tools-chaos-key"
_OWNED_CHAT: Final = "malformed-tools-owned-chat"
_OWNED_STORE: Final = "vs_malformed_tools_owned_store"
_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})")
_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
_OWNED_PROXY_CELL_SECONDS: Final = 2 * graceful_stop_seconds() + 120
_RESPAWN_SECONDS: Final = graceful_stop_seconds() + 60
_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1")))
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_NOT_A_LIST: Final = "tools: Input should be a valid list"
_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
}
_CHAT_TOOL: Final[dict[str, JsonValue]] = {
"type": "function",
"function": {"name": "get_weather", "description": "Look up the weather", "parameters": _WEATHER_SCHEMA},
}
_CLAUDE_TOOL: Final[dict[str, JsonValue]] = {
"name": "get_weather",
"description": "Look up the weather",
"input_schema": _WEATHER_SCHEMA,
}
_RESPONSES_TOOL: Final[dict[str, JsonValue]] = {
"type": "function",
"name": "get_weather",
"description": "Look up the weather",
"parameters": _WEATHER_SCHEMA,
}
_GEMINI_TOOL: Final[dict[str, JsonValue]] = {
"functionDeclarations": [
{"name": "get_weather", "description": "Look up the weather", "parameters": _WEATHER_SCHEMA}
]
}
Kind: TypeAlias = Literal["valid", "rejected", "dropped"]
def _marker(label: str) -> str:
return f"{label} marker-{uuid.uuid4().hex}"
def _tag(text: str) -> str:
found: Final = _MARKER.search(text)
return found.group(1) if found else "none"
def _answer(tag: str) -> str:
return f"answer {tag}"
def _gemini_body(tag: str) -> bytes:
return json.dumps(
{
"candidates": [
{"content": {"role": "model", "parts": [{"text": _answer(tag)}]}, "index": 0, "finishReason": "STOP"}
],
"usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 2, "totalTokenCount": 11},
"modelVersion": _GEMINI_BACKEND,
}
).encode()
def _scripted(request: Request) -> Reply:
text: Final = request.body.decode()
tag: Final = _tag(text)
path: Final = request.target.split("?")[0]
stream: Final = bool(text) and _JSON_OBJECT.validate_json(request.body).get("stream") is True
if path == "/v1/chat/completions":
return chat_reply(f"chatcmpl-{tag}", _OPENAI_BACKEND, _answer(tag), stream=stream)
if path == "/v1/responses":
return responses_reply(f"resp_{tag}", _OPENAI_BACKEND, _answer(tag), stream=stream)
if path == "/v1/messages" and _JSON_OBJECT.validate_json(request.body).get("tools") == 5:
return Reply(status=400, body=error_body(400, _NOT_A_LIST))
if path == "/v1/messages" and stream:
return Reply(
content_type="text/event-stream", chunks=message_stream(f"msg_{tag}", _CLAUDE_BACKEND, _answer(tag))
)
if path == "/v1/messages":
return Reply(body=message_json(f"msg_{tag}", _CLAUDE_BACKEND, _answer(tag)))
if path.endswith(":streamGenerateContent"):
return Reply(content_type="text/event-stream", chunks=(b"data: " + _gemini_body(tag) + b"\r\n\r\n",))
if path.endswith(":generateContent"):
return Reply(body=_gemini_body(tag))
return Reply(status=404, body=json.dumps({"error": f"unscripted {request.target}"}).encode())
@dataclass(frozen=True, slots=True)
class _Switchboard:
down: threading.Event
holding: threading.Event
release: threading.Event
held: SimpleQueue[str]
def respond(self, request: Request) -> Reply:
if self.down.is_set():
return Reply(drop_connection=True)
if self.holding.is_set():
self.held.put(request.target)
assert self.release.wait(timeout=120), "Held request was never released"
return _scripted(request)
def reset(self) -> None:
self.down.clear()
self.release.set()
self.holding.clear()
@dataclass(frozen=True, slots=True)
class _Rig:
gateway: Gateway
wire: Wire
switchboard: _Switchboard
chat: str
claude: str
gemini: str
@pytest.fixture(scope="module")
def rig() -> Iterator[_Rig]:
switchboard: Final = _Switchboard(threading.Event(), threading.Event(), threading.Event(), SimpleQueue())
with (
gateway_from_environment() as gateway,
wire_server(switchboard.respond) as wire,
gateway.scenario() as scenario,
):
built: Final = _Rig(
gateway=gateway,
wire=wire,
switchboard=switchboard,
chat=scenario.model(api_base=f"{wire.url}/v1", api_key=_PROVIDER_KEY),
claude=scenario.model(model=f"anthropic/{_CLAUDE_BACKEND}", api_base=wire.url, api_key=_PROVIDER_KEY),
gemini=scenario.model(model=f"gemini/{_GEMINI_BACKEND}", api_base=wire.url, api_key=_PROVIDER_KEY),
)
for group in (built.chat, built.claude, built.gemini):
wait_for_worker_readiness(gateway, group, 1, _PROXY_WORKERS)
try:
yield built
finally:
switchboard.reset()
@pytest.fixture(autouse=True)
def _reset(rig: _Rig) -> Iterator[None]:
rig.switchboard.reset()
rig.wire.drain()
yield
rig.switchboard.reset()
@dataclass(frozen=True, slots=True)
class _Call:
path: str
body: dict[str, JsonValue]
marker: str
kind: Kind
def _tools_for(kind: Kind, tool: dict[str, JsonValue]) -> dict[str, JsonValue]:
if kind == "rejected":
return {"tools": 5}
if kind == "dropped":
return {"tools": [tool, 5], "drop_params": True}
return {"tools": [tool]}
def _call(rig: _Rig, index: int, kind: Kind, label: str) -> _Call:
marker: Final = _marker(label)
stream: Final = (index // 4) % 2 == 1
family: Final = index % 4
if family == 0:
return _Call(
"/v1/chat/completions",
{
"model": rig.chat,
"messages": [{"role": "user", "content": marker}],
"stream": stream,
**_tools_for(kind, _CHAT_TOOL),
},
marker,
kind,
)
if family == 1:
return _Call(
"/v1/messages",
{
"model": rig.claude,
"max_tokens": 16,
"messages": [{"role": "user", "content": marker}],
"stream": stream,
**_tools_for(kind, _CLAUDE_TOOL),
},
marker,
kind,
)
if family == 2:
return _Call(
"/v1/responses",
{"model": rig.chat, "input": marker, "stream": stream, **_tools_for(kind, _RESPONSES_TOOL)},
marker,
kind,
)
action: Final = "streamGenerateContent?alt=sse" if stream else "generateContent"
return _Call(
f"/v1beta/models/{rig.gemini}:{action}",
{"contents": [{"role": "user", "parts": [{"text": marker}]}], **_tools_for(kind, _GEMINI_TOOL)},
marker,
kind,
)
def _kind(index: int) -> Kind:
kinds: Final[tuple[Kind, ...]] = ("valid", "rejected", "dropped")
return kinds[index % 3]
def _clients(stack: ExitStack, base_url: str, count: int) -> tuple[httpx.Client, ...]:
return tuple(
stack.enter_context(httpx.Client(base_url=base_url, timeout=120, trust_env=False)) for _ in range(count)
)
def _post(client: httpx.Client, key: str, call: _Call) -> httpx.Response:
return client.post(call.path, json=call.body, headers={"Authorization": f"Bearer {key}"})
def _status_or_dropped(client: httpx.Client, key: str, call: _Call) -> int | None:
try:
return _post(client, key, call).status_code
except httpx.TransportError:
return None
def _deliveries(wire: Wire) -> tuple[str, ...]:
return tuple(request.body.decode() for request in wire.drain())
def _seen(deliveries: tuple[str, ...], marker: str) -> int:
return sum(marker in body for body in deliveries)
def _delivered_tool_names(deliveries: tuple[str, ...], marker: str) -> tuple[str, ...]:
(body,) = tuple(body for body in deliveries if marker in body)
return tuple(re.findall(r'"name": ?"([a-z_]+)"', body))
def _assert_wave(calls: tuple[_Call, ...], statuses: tuple[int | None, ...], deliveries: tuple[str, ...]) -> None:
expected_statuses: Final = tuple(400 if call.kind == "rejected" else 200 for call in calls)
assert statuses == expected_statuses, tuple(zip(statuses, (call.marker for call in calls), strict=True))
seen: Final = tuple(_seen(deliveries, call.marker) for call in calls)
assert seen == tuple(0 if call.kind == "rejected" else 1 for call in calls), seen
names: Final = tuple(_delivered_tool_names(deliveries, call.marker) for call in calls if call.kind != "rejected")
assert all("get_weather" in found for found in names), names
def test_mixed_burst_rejects_malformed_tools_and_forwards_the_rest_once(rig: _Rig) -> None:
with ExitStack() as stack:
calls: Final = tuple(_call(rig, index, _kind(index), "burst") for index in range(40))
clients: Final = _clients(stack, str(rig.gateway.client.base_url), len(calls))
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(calls)))
statuses: Final = tuple(
pool.map(
lambda pair: _status_or_dropped(pair[0], rig.gateway.key, pair[1]), zip(clients, calls, strict=True)
)
)
_assert_wave(calls, statuses, _deliveries(rig.wire))
def test_upstream_outage_still_rejects_malformed_tools_and_recovers(rig: _Rig) -> None:
with ExitStack() as stack:
clients: Final = _clients(stack, str(rig.gateway.client.base_url), 12)
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients)))
def wave(label: str) -> tuple[tuple[_Call, ...], tuple[int | None, ...]]:
calls: Final = tuple(_call(rig, index, _kind(index), label) for index in range(len(clients)))
statuses: Final = tuple(
pool.map(
lambda pair: _status_or_dropped(pair[0], rig.gateway.key, pair[1]), zip(clients, calls, strict=True)
)
)
return calls, statuses
before_calls, before_statuses = wave("before outage")
_assert_wave(before_calls, before_statuses, _deliveries(rig.wire))
rig.switchboard.down.set()
outage_calls, outage_statuses = wave("during outage")
for call, status in zip(outage_calls, outage_statuses, strict=True):
assert (status == 400) if call.kind == "rejected" else (status is not None and status >= 500), (
call.kind,
status,
)
assert rig.gateway.request("GET", "/health/liveliness").status_code == 200
rig.wire.drain()
rig.switchboard.down.clear()
after_calls, after_statuses = wave("after outage")
_assert_wave(after_calls, after_statuses, _deliveries(rig.wire))
def test_held_upstream_does_not_delay_malformed_tools_rejections(rig: _Rig) -> None:
with ExitStack() as stack:
held_calls: Final = tuple(_call(rig, index * 4, "valid", "held") for index in range(8))
rejected_calls: Final = tuple(_call(rig, index, "rejected", "rejected") for index in range(8))
clients: Final = _clients(stack, str(rig.gateway.client.base_url), len(held_calls) + len(rejected_calls))
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients)))
rig.switchboard.release.clear()
rig.switchboard.holding.set()
stack.callback(rig.switchboard.release.set)
pending: Final[tuple[Future[int | None], ...]] = tuple(
pool.submit(_status_or_dropped, client, rig.gateway.key, call)
for client, call in zip(clients[: len(held_calls)], held_calls, strict=True)
)
eventually(rig.switchboard.held.qsize, lambda size: size == len(held_calls), seconds=60)
rejected: Final = tuple(
pool.map(
lambda pair: _status_or_dropped(pair[0], rig.gateway.key, pair[1]),
zip(clients[len(held_calls) :], rejected_calls, strict=True),
)
)
assert rejected == (400,) * len(rejected_calls), rejected
assert not any(future.done() for future in pending), "a held request finished before its release"
rig.switchboard.release.set()
assert tuple(future.result(timeout=120) for future in pending) == (200,) * len(held_calls)
deliveries: Final = _deliveries(rig.wire)
assert tuple(_seen(deliveries, call.marker) for call in held_calls) == (1,) * len(held_calls)
assert tuple(_seen(deliveries, call.marker) for call in rejected_calls) == (0,) * len(rejected_calls)
@dataclass(frozen=True, slots=True)
class _Owned:
proxy: OwnedProxy
wire: Wire
held: SimpleQueue[str]
holding: threading.Event
release: threading.Event
blocked_tool: str
denied_tool: str
def _owned_config(path: Path, upstream: str, denied_tool: str) -> Path:
path.write_text(
json.dumps(
{
"model_list": [
{
"model_name": _OWNED_CHAT,
"litellm_params": {
"model": f"openai/{_OPENAI_BACKEND}",
"api_base": f"{upstream}/v1",
"api_key": _PROVIDER_KEY,
},
}
],
"general_settings": {
"master_key": "os.environ/LITELLM_MASTER_KEY",
"database_url": "os.environ/DATABASE_URL",
},
"litellm_settings": {"drop_params": True},
"router_settings": {"disable_cooldowns": True},
"guardrails": [
{
"guardrail_name": "malformed-tools-policy",
"litellm_params": {"guardrail": "tool_policy", "mode": "pre_call", "default_on": True},
},
{
"guardrail_name": "malformed-tools-permission",
"litellm_params": {
"guardrail": "tool_permission",
"mode": "pre_call",
"default_on": False,
"default_action": "allow",
"on_disallowed_action": "block",
"rules": [{"id": "deny_tool", "tool_name": f"^{denied_tool}$", "decision": "deny"}],
},
},
],
"vector_store_registry": [
{
"vector_store_name": "malformed-tools-owned-store",
"litellm_params": {
"vector_store_id": _OWNED_STORE,
"custom_llm_provider": "openai",
"api_base": upstream,
"api_key": _PROVIDER_KEY,
},
}
],
}
)
)
return path
@pytest.fixture(scope="module")
def owned(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Owned]:
suffix: Final = uuid.uuid4().hex[:12]
blocked_tool: Final = f"wipe_disk_{suffix}"
denied_tool: Final = f"run_shell_{suffix}"
held: Final[SimpleQueue[str]] = SimpleQueue()
holding: Final = threading.Event()
release: Final = threading.Event()
def respond(request: Request) -> Reply:
if holding.is_set():
held.put(request.target)
assert release.wait(timeout=120), "Held request was never released"
return _scripted(request)
directory: Final = tmp_path_factory.mktemp("malformed-tools-owned")
with ExitStack() as stack:
gateway: Final = stack.enter_context(gateway_from_environment())
policy: Final = gateway.request(
"POST", "/v1/tool/policy", {"tool_name": blocked_tool, "input_policy": "blocked"}
)
assert policy.status_code == 200, policy.text
stack.callback(write_rows, 'DELETE FROM "LiteLLM_ToolTable" WHERE tool_name = %s', (blocked_tool,))
wire: Final = stack.enter_context(wire_server(respond))
stack.callback(release.set)
proxy: Final = stack.enter_context(
owned_proxy_process(
gateway,
directory,
{"ANTHROPIC_API_BASE": wire.url, "ANTHROPIC_API_KEY": _PROVIDER_KEY},
config=_owned_config(directory / "malformed-tools.yaml", wire.url, denied_tool),
workers=2,
)
)
yield _Owned(proxy, wire, held, holding, release, blocked_tool, denied_tool)
def _owned_chat(owned: _Owned, marker: str, **fields: JsonValue) -> httpx.Response:
return owned.proxy.gateway.request(
"POST",
"/v1/chat/completions",
{"model": _OWNED_CHAT, "messages": [{"role": "user", "content": marker}], **fields},
)
def _named_tool(name: str) -> dict[str, JsonValue]:
return {"type": "function", "function": {"name": name, "parameters": _WEATHER_SCHEMA}}
@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
def test_global_drop_params_wins_over_an_explicit_request_false(owned: _Owned) -> None:
marker: Final = _marker("global drop")
response: Final = _owned_chat(owned, marker, tools=5, drop_params=False)
assert response.status_code == 200, response.text
assert _answer(_tag(marker)) in response.text, response.text
(sent,) = tuple(request for request in owned.wire.drain() if marker in request.body.decode())
assert "tools" not in _JSON_OBJECT.validate_json(sent.body), sent.body
@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
def test_tool_policy_still_blocks_a_tool_kept_beside_a_dropped_entry(owned: _Owned) -> None:
marker: Final = _marker("tool policy")
response: Final = _owned_chat(owned, marker, tools=[_named_tool(owned.blocked_tool), 5])
assert response.status_code == 400, response.text
assert "Violated tool policy" in response.text and owned.blocked_tool in response.text, response.text
assert tuple(request for request in owned.wire.drain() if marker in request.body.decode()) == ()
@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
def test_tool_permission_still_denies_a_tool_kept_beside_a_dropped_entry(owned: _Owned) -> None:
marker: Final = _marker("tool permission")
response: Final = _owned_chat(
owned, marker, tools=[_named_tool(owned.denied_tool), "x"], guardrails=["malformed-tools-permission"]
)
assert response.status_code == 400, response.text
assert "Violated guardrail policy" in response.text and owned.denied_tool in response.text, response.text
assert tuple(request for request in owned.wire.drain() if marker in request.body.decode()) == ()
@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
def test_anthropic_pass_through_forwards_malformed_tools_unchanged(owned: _Owned) -> None:
marker: Final = _marker("pass-through")
body: Final[dict[str, JsonValue]] = {
"model": _CLAUDE_BACKEND,
"max_tokens": 16,
"messages": [{"role": "user", "content": marker}],
"tools": 5,
}
response: Final = owned.proxy.gateway.request("POST", "/anthropic/v1/messages", body)
assert response.status_code == 400, response.text
assert _JSON_OBJECT.validate_json(response.content) == _JSON_OBJECT.validate_json(error_body(400, _NOT_A_LIST))
(sent,) = tuple(request for request in owned.wire.drain() if marker in request.body.decode())
assert sent.target == "/v1/messages", sent.target
assert _JSON_OBJECT.validate_json(sent.body) == body, sent.body
@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
def test_anthropic_pass_through_streams_well_formed_tools(owned: _Owned) -> None:
marker: Final = _marker("pass-through stream")
body: Final[dict[str, JsonValue]] = {
"model": _CLAUDE_BACKEND,
"max_tokens": 16,
"messages": [{"role": "user", "content": marker}],
"tools": [_CLAUDE_TOOL],
"stream": True,
}
response: Final = owned.proxy.gateway.request("POST", "/anthropic/v1/messages", body)
assert response.status_code == 200, response.text
assert response.content == b"".join(message_stream(f"msg_{_tag(marker)}", _CLAUDE_BACKEND, _answer(_tag(marker))))
(sent,) = tuple(request for request in owned.wire.drain() if marker in request.body.decode())
assert _JSON_OBJECT.validate_json(sent.body) == body, sent.body
def _local_port(client: httpx.Client) -> int:
with client.stream("GET", "/health/liveliness") as response:
port: Final = int(response.extensions["network_stream"].get_extra_info("client_addr")[1])
response.read()
assert response.status_code == 200, response.text
return port
def _accepted_client_ports(pid: int, proxy_port: int) -> frozenset[int]:
return frozenset(
connection.raddr.port
for connection in psutil.Process(pid).net_connections(kind="tcp")
if connection.raddr and connection.laddr.port == proxy_port
)
def _owned_call(index: int, label: str) -> _Call:
marker: Final = _marker(label)
tools: Final[JsonValue] = [_CHAT_TOOL, 5] if index % 2 else [_CHAT_TOOL]
return _Call(
"/v1/chat/completions",
{
"model": _OWNED_CHAT,
"messages": [{"role": "user", "content": marker}],
"stream": index % 4 == 1,
"tools": tools,
},
marker,
"dropped" if index % 2 else "valid",
)
@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
def test_worker_sigkill_mid_burst_leaves_the_replacement_dropping_bad_entries(owned: _Owned) -> None:
proxy_url: Final = owned.proxy.gateway.client.base_url
workers: Final = eventually(
lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.proxy.log.read_text())),
lambda pids: len(pids) >= 2,
seconds=30,
)
started_before: Final = len(workers)
serving: Final = owned.proxy.worker_pids()
owned.wire.drain()
with ExitStack() as stack:
clients: Final = _clients(stack, str(proxy_url), 12)
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients)))
ports: Final = tuple(_local_port(client) for client in clients)
calls: Final = tuple(_owned_call(index, "worker kill") for index in range(len(clients)))
owned.release.clear()
owned.holding.set()
stack.callback(owned.holding.clear)
stack.callback(owned.release.set)
futures: Final = tuple(
pool.submit(_status_or_dropped, client, owned.proxy.gateway.key, call)
for client, call in zip(clients, calls, strict=True)
)
eventually(owned.held.qsize, lambda size: size == len(clients), seconds=60)
shares: Final = {pid: _accepted_client_ports(pid, proxy_url.port or 0) & frozenset(ports) for pid in serving}
assert sum(map(len, shares.values())) == len(clients), shares
victim: Final = min((pid for pid in serving if shares[pid]), key=lambda pid: len(shares[pid]))
with owned.proxy.replacing_worker(victim) as kill:
kill()
owned.holding.clear()
owned.release.set()
results: Final = tuple(future.result(timeout=120) for future in futures)
for port, result in zip(ports, results, strict=True):
assert result == (None if port in shares[victim] else 200), (port, result, shares)
eventually(
lambda: len(_STARTED_WORKER.findall(owned.proxy.log.read_text())),
lambda started: started > started_before,
_RESPAWN_SECONDS,
)
second_wave: Final = tuple(_owned_call(index * 2 + 1, "after kill") for index in range(6))
(fresh,) = _clients(stack, str(proxy_url), 1)
assert tuple(_post(fresh, owned.proxy.gateway.key, call).status_code for call in second_wave) == (200,) * len(
second_wave
)
deliveries: Final = _deliveries(owned.wire)
everything: Final = calls + second_wave
assert tuple(_seen(deliveries, call.marker) for call in everything) == (1,) * len(everything)
assert tuple(_delivered_tool_names(deliveries, call.marker) for call in everything) == (
("get_weather",),
) * len(everything)
assert owned.proxy.process.poll() is None

File diff suppressed because it is too large Load diff

View file

@ -1,21 +1,19 @@
import json
import re
import threading
import uuid
from collections.abc import Callable, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import ExitStack
from hashlib import sha256
from pathlib import Path
from typing import Final
import httpx
import psutil
import pytest
import yaml
from integration._support.client import Gateway, eventually, held, object_value
from integration._support.database import read_rows
from integration._support.process import graceful_stop_seconds, owned_proxy_process
from integration._support.process import WorkerHealthcheck, graceful_stop_seconds, owned_proxy_process
from integration._support.vertex import service_account_json
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
@ -38,7 +36,6 @@ _GEMINI_USAGE: Final = {
"totalTokenCount": _PROMPT_TOKENS + _COMPLETION_TOKENS,
}
_ZERO_USAGE: Final = {"promptTokenCount": 0, "candidatesTokenCount": 0, "totalTokenCount": 0}
_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
_BURST_SELECT: Final = (
'SELECT api_key, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs"'
" WHERE api_key = ANY(string_to_array(%s, ','))"
@ -67,6 +64,7 @@ _UPSTREAM_ASKS_FOR_USAGE: Final = {
},
}
_SETTING_VALUES: Final = {"always_off": False, "always_on": True}
_STREAMS_PER_WORKER: Final = 6
def _openai_frame(chunk: Mapping[str, JsonValue], response_id: str) -> bytes:
@ -273,20 +271,15 @@ def _stream_or_dropped(client: httpx.Client, key: str) -> httpx.Response | None:
return None
def _local_port(client: httpx.Client) -> int:
with client.stream("GET", "/health/liveliness") as response:
port: Final = int(response.extensions["network_stream"].get_extra_info("client_addr")[1])
response.read()
assert response.status_code == 200, response.text
return port
def _accepted_client_ports(pid: int, proxy_port: int) -> frozenset[int]:
return frozenset(
connection.raddr.port
for connection in psutil.Process(pid).net_connections(kind="tcp")
if connection.raddr and connection.laddr.port == proxy_port
def _held_streams(
stack: ExitStack, pool: ThreadPoolExecutor, url: str, keys: Sequence[str], wire: Wire, held: int
) -> tuple[Future[httpx.Response | None], ...]:
clients: Final = tuple(stack.enter_context(httpx.Client(base_url=url, timeout=60, trust_env=False)) for _ in keys)
futures: Final = tuple(
pool.submit(_stream_or_dropped, client, key) for client, key in zip(clients, keys, strict=True)
)
eventually(wire.received.qsize, lambda size: size >= held, seconds=60)
return futures
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
@ -296,35 +289,27 @@ def test_worker_killed_mid_stream_leaves_the_sibling_billing_each_stream_once(ga
wire: Final = stack.enter_context(wire_server(_peer(gate)))
stack.callback(gate.set)
config: Final = _owned_config(tmp_path / "worker-kill.yaml", gateway, wire, {})
owned: Final = stack.enter_context(owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2))
workers: Final = eventually(
lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())),
lambda pids: len(pids) == 2,
seconds=30,
owned: Final = stack.enter_context(
owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2, healthcheck=WorkerHealthcheck.LENIENT)
)
victim, survivor = sorted(eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=30))
scenario: Final = stack.enter_context(owned.gateway.scenario())
proxy_url: Final = owned.gateway.client.base_url
clients: Final = tuple(
stack.enter_context(httpx.Client(base_url=str(proxy_url), timeout=60, trust_env=False)) for _ in range(12)
)
keys: Final = tuple(scenario.key(models=[_ADAPTER_MODEL]) for _ in clients)
ports: Final = tuple(_local_port(client) for client in clients)
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients)))
futures: Final = tuple(
pool.submit(_stream_or_dropped, client, key) for client, key in zip(clients, keys, strict=True)
)
eventually(wire.received.qsize, lambda size: size >= len(clients), seconds=60)
shares: Final = {pid: _accepted_client_ports(pid, proxy_url.port or 0) & frozenset(ports) for pid in workers}
assert sum(map(len, shares.values())) == len(clients), shares
victim: Final = min((pid for pid in workers if shares[pid]), key=lambda pid: len(shares[pid]))
victim_keys: Final = tuple(scenario.key(models=[_ADAPTER_MODEL]) for _ in range(_STREAMS_PER_WORKER))
survivor_keys: Final = tuple(scenario.key(models=[_ADAPTER_MODEL]) for _ in range(_STREAMS_PER_WORKER))
proxy_url: Final = str(owned.gateway.client.base_url)
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=2 * _STREAMS_PER_WORKER))
with owned.suspended(survivor):
on_victim: Final = _held_streams(stack, pool, proxy_url, victim_keys, wire, _STREAMS_PER_WORKER)
with owned.suspended(victim):
on_survivor: Final = _held_streams(stack, pool, proxy_url, survivor_keys, wire, 2 * _STREAMS_PER_WORKER)
with owned.replacing_worker(victim) as kill:
kill()
gate.set()
results: Final = tuple(future.result(timeout=60) for future in futures)
survivors: Final = tuple(
(key, result) for port, key, result in zip(ports, keys, results, strict=True) if port not in shares[victim]
)
assert survivors, shares
dropped: Final = tuple(future.result(timeout=60) for future in on_victim)
survivors: Final = tuple(
zip(survivor_keys, (future.result(timeout=60) for future in on_survivor), strict=True)
)
assert dropped == (None,) * _STREAMS_PER_WORKER, dropped
for _, result in survivors:
assert result is not None
_assert_one_stop_with(result, _GEMINI_USAGE)

View file

@ -379,48 +379,47 @@ def test_a_replacing_block_that_never_kills_its_worker_fails_and_leaves_it_servi
assert owned.worker_pids() == frozenset((worker,))
def _kill_of(owned: OwnedProxy, worker: int) -> WorkerKill:
return WorkerKill(psutil.Process(worker), psutil.Process(owned.process.pid), 5, threading.Event())
@contextmanager
def _unreaped_child() -> Iterator[psutil.Process]:
with subprocess.Popen((sys.executable, "-I", "-c", "import time; time.sleep(60)")) as child:
try:
yield psutil.Process(child.pid)
finally:
child.kill()
def test_a_worker_kill_returns_once_the_supervisor_reaped_the_victim_and_cannot_fire_twice(tmp_path: Path) -> None:
with _supervised(tmp_path, WorkerHealthcheck.STRICT) as (owned, worker):
kill: Final = _kill_of(owned, worker)
def test_a_worker_kill_returns_once_the_victim_is_dead_before_anything_reaps_it_and_cannot_fire_twice() -> None:
with _unreaped_child() as victim:
kill: Final = WorkerKill(victim, threading.Event())
kill()
assert not psutil.pid_exists(worker)
assert victim.status() == psutil.STATUS_ZOMBIE
assert kill.fired.is_set()
with pytest.raises(AssertionError, match="already killed"):
kill()
def test_a_frozen_victim_stays_stopped_until_the_kill(tmp_path: Path) -> None:
with _supervised(tmp_path, WorkerHealthcheck.STRICT) as (owned, worker):
kill: Final = _kill_of(owned, worker)
def test_a_frozen_victim_stays_stopped_until_the_kill() -> None:
with _unreaped_child() as victim:
kill: Final = WorkerKill(victim, threading.Event())
assert kill.freeze().status() == psutil.STATUS_STOPPED
kill()
assert not psutil.pid_exists(worker)
assert victim.status() == psutil.STATUS_ZOMBIE
def test_a_kill_under_a_suspended_supervisor_returns_with_the_victim_left_unreaped(tmp_path: Path) -> None:
with _supervised(tmp_path, WorkerHealthcheck.LENIENT) as (owned, worker):
kill: Final = _kill_of(owned, worker)
with owned.suspended(owned.process.pid):
kill()
assert psutil.Process(worker).status() == psutil.STATUS_ZOMBIE
eventually(lambda: psutil.pid_exists(worker), lambda exists: not exists, seconds=5)
def _kill_after_freezing_the_supervisor(owned: OwnedProxy, worker: int, supervisor: psutil.Process) -> None:
with owned.replacing_worker(worker) as kill:
supervisor.suspend()
kill()
def test_a_kill_fails_when_the_supervisor_never_reaps_the_victim(tmp_path: Path) -> None:
def test_a_replacing_block_fails_when_the_supervisor_never_reaps_the_killed_worker(tmp_path: Path) -> None:
with _supervised(tmp_path, WorkerHealthcheck.STRICT) as (owned, worker):
unreaping: Final = psutil.Process(owned.process.pid)
kill: Final = WorkerKill(psutil.Process(worker), psutil.Process(os.getpid()), 0.5, threading.Event())
unreaping.suspend()
try:
with pytest.raises(AssertionError, match="did not reap"):
kill()
_kill_after_freezing_the_supervisor(owned, worker, unreaping)
finally:
unreaping.resume()
assert not kill.fired.is_set()
def test_a_strict_healthcheck_proxy_refuses_suspension_before_stopping_anything(tmp_path: Path) -> None:

View file

@ -9,12 +9,14 @@ response. A failure raised after the router picked a deployment (a post-call
guardrail block, a provider error) carries no flag and keeps its deployment labels.
"""
from typing import Final
import pytest
from fastapi import HTTPException
from prometheus_client import REGISTRY
from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY
from litellm.exceptions import GuardrailRaisedException
from litellm.exceptions import GuardrailRaisedException, MalformedRequestFieldError
from litellm.integrations.prometheus import PrometheusLogger
from litellm.proxy._types import ProxyException
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
@ -99,3 +101,15 @@ def test_failure_after_routing_still_marks_deployment_partial_outage(failure: Ex
assert _model_id_values(logger.litellm_deployment_state) == {"dep-1"}
assert _model_id_values(logger.litellm_deployment_failure_responses) == {"dep-1"}
def test_malformed_field_rejection_after_routing_leaves_deployment_healthy() -> None:
logger: Final = PrometheusLogger()
rejection: Final = MalformedRequestFieldError(
message="tools: Input should be a valid list", model="internal-model", llm_provider="openai"
)
logger.set_llm_deployment_failure_metrics(_attributed_failure_kwargs(rejection, rejected_before_routing=False))
assert logger.litellm_deployment_state._metrics == {}
assert _model_id_values(logger.litellm_deployment_failure_responses) == {""}

View file

@ -1,4 +1,6 @@
from typing import Final
import pytest
import litellm
@ -49,11 +51,12 @@ def test_web_search_cost_high():
)
def test_get_web_search_options_preserves_explicit_none_error():
with pytest.raises(TypeError):
StandardBuiltInToolCostTracking.get_web_search_options(
{"web_search_options": None, "tools": [{"type": "web_search_preview"}]}
)
def test_get_web_search_options_reads_tools_when_options_are_null() -> None:
options: Final = StandardBuiltInToolCostTracking.get_web_search_options(
{"web_search_options": None, "tools": [{"type": "web_search_preview"}]}
)
assert options == {"type": "web_search_preview"}
def test_get_web_search_options_accepts_non_list_tool_iterables():
@ -1070,3 +1073,13 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None:
usage = Usage(prompt_tokens=1840, completion_tokens=412, total_tokens=2252)
assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None
@pytest.mark.parametrize(
"kwargs",
[{"tools": 5}, {"web_search_options": None}, {"web_search_options": 5}],
ids=["int_tools", "null_web_search_options", "int_web_search_options"],
)
def test_built_in_tool_lookups_ignore_malformed_request_values(kwargs: dict[str, object]) -> None:
assert StandardBuiltInToolCostTracking.get_web_search_options(kwargs) is None
assert StandardBuiltInToolCostTracking.get_file_search_tool_call(kwargs) is None

View file

@ -0,0 +1,187 @@
import datetime
from collections.abc import Callable, Mapping
from typing import Final
import pytest
import litellm
from litellm import Router
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.utils import async_pre_call_deployment_hook
_GOOD_TOOL: Final = {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}
_MALFORMED_TOOLS: Final = (
5,
"get_weather",
["get_weather"],
{"type": "function"},
[{"type": "function", "function": {"name": "get_weather", "parameters": "object"}}],
)
def _deployment(name: str, **params: object) -> Mapping[str, object]:
return {
"model_name": name,
"litellm_params": {"model": "openai/gpt-5.2", "api_key": "sk-test", "mock_response": f"served by {name}", **params},
}
def _proxy_request(model: str, route: str = "acompletion", **fields: object) -> Mapping[str, object]:
return {
"model": model,
"messages": [{"role": "user", "content": "weather in Paris?"}],
"proxy_server_request": {"url": "/v1/chat/completions", "body": {}},
"litellm_logging_obj": Logging(
model=model,
messages=[],
stream=False,
call_type=route,
start_time=datetime.datetime.now(),
litellm_call_id="call-1",
function_id="fn-1",
),
"num_retries": 0,
**fields,
}
@pytest.mark.parametrize("tools", _MALFORMED_TOOLS)
@pytest.mark.asyncio
async def test_malformed_tools_fall_back_to_a_deployment_that_drops_them(tools: object) -> None:
router: Final = Router(
model_list=[_deployment("primary"), _deployment("backup", additional_drop_params=["tools"])],
fallbacks=[{"primary": ["backup"]}],
)
response: Final = await router.acompletion(**_proxy_request("primary", tools=tools))
assert response.choices[0].message.content == "served by backup"
@pytest.mark.asyncio
async def test_fallback_on_unsupported_params_false_keeps_the_400() -> None:
router: Final = Router(
model_list=[_deployment("primary"), _deployment("backup", additional_drop_params=["tools"])],
fallbacks=[{"primary": ["backup"]}],
fallback_on_unsupported_params=False,
)
with pytest.raises(litellm.BadRequestError):
await router.acompletion(**_proxy_request("primary", tools=5))
@pytest.mark.asyncio
async def test_a_rejected_request_cools_down_no_deployment() -> None:
cools_on_any_400: Final = {"allowed_fails_policy": {"BadRequestErrorAllowedFails": 0}, "cooldown_time": 120}
router: Final = Router(
model_list=[
{**_deployment("primary"), "model_info": {"id": "primary", **cools_on_any_400}},
{**_deployment("backup"), "model_info": {"id": "backup", **cools_on_any_400}},
],
fallbacks=[{"primary": ["backup"]}],
)
with pytest.raises(litellm.BadRequestError):
await router.acompletion(**_proxy_request("primary", tools=5))
primary: Final = await router.acompletion(**_proxy_request("primary"))
backup: Final = await router.acompletion(**_proxy_request("backup"))
assert (primary.choices[0].message.content, backup.choices[0].message.content) == (
"served by primary",
"served by backup",
)
@pytest.mark.parametrize("tools", _MALFORMED_TOOLS)
@pytest.mark.parametrize(
"deployment",
[_deployment("keeps"), _deployment("keeps", tools=[_GOOD_TOOL])],
ids=["plain", "with_configured_tools"],
)
@pytest.mark.asyncio
async def test_malformed_tools_answer_400_when_the_served_deployment_keeps_tools(
tools: object, deployment: Mapping[str, object]
) -> None:
router: Final = Router(model_list=[deployment])
with pytest.raises(litellm.BadRequestError) as raised:
await router.acompletion(**_proxy_request("keeps", tools=tools))
assert (raised.value.status_code, raised.value.param, raised.value.type) == (
400,
"tools",
"invalid_request_error",
)
@pytest.mark.asyncio
async def test_a_request_drop_params_false_overrides_a_deployment_that_drops() -> None:
router: Final = Router(model_list=[_deployment("drops", drop_params=True)])
with pytest.raises(litellm.BadRequestError):
await router.acompletion(**_proxy_request("drops", tools=5, drop_params=False))
@pytest.mark.parametrize(
"route", ["acompletion", "aresponses", "anthropic_messages", "agenerate_content", "acreate_interaction"]
)
@pytest.mark.asyncio
async def test_drop_params_drops_every_malformed_tool_field(route: str) -> None:
request: Final = _proxy_request(
"drops", route, drop_params=True, tools=5, tool_choice="auto", web_search_options="high"
)
sent: Final = await async_pre_call_deployment_hook(dict(request), route)
assert {"tools", "tool_choice", "web_search_options"}.isdisjoint(sent)
assert sent["messages"] == request["messages"]
@pytest.mark.asyncio
async def test_additional_drop_params_drops_only_the_malformed_tool_entries() -> None:
request: Final = _proxy_request("drops", additional_drop_params=["tools"], tools=[_GOOD_TOOL, "get_weather"])
sent: Final = await async_pre_call_deployment_hook(dict(request), "acompletion")
assert sent["tools"] == [_GOOD_TOOL]
@pytest.mark.asyncio
async def test_an_sdk_call_keeps_the_tools_it_passed() -> None:
tools: Final = (_GOOD_TOOL,)
request: Final = {
key: value for key, value in _proxy_request("sdk", tools=tools).items() if key != "proxy_server_request"
}
sent: Final = await async_pre_call_deployment_hook(dict(request), "acompletion")
assert sent["tools"] is tools
@pytest.mark.parametrize("schema", ["", [], False, 0, {}, '{"type": "object"}'])
@pytest.mark.parametrize(
"tool",
[
lambda schema: {"type": "function", "function": {"name": "get_weather", "parameters": schema}},
lambda schema: {"type": "function", "name": "get_weather", "parameters": schema},
lambda schema: {"name": "get_weather", "input_schema": schema},
],
ids=["chat", "responses", "messages"],
)
@pytest.mark.asyncio
async def test_empty_and_json_string_schemas_reach_the_deployment(
tool: Callable[[object], Mapping[str, object]], schema: object
) -> None:
tools: Final = [tool(schema)]
sent: Final = await async_pre_call_deployment_hook(dict(_proxy_request("keeps", tools=tools)), "acompletion")
assert sent["tools"] is tools
@pytest.mark.parametrize("tools", [(_GOOD_TOOL,), [{"type": "web_search", "function": None}]], ids=["tuple", "null"])
@pytest.mark.asyncio
async def test_tools_core_accepts_reach_the_deployment(tools: object) -> None:
sent: Final = await async_pre_call_deployment_hook(dict(_proxy_request("keeps", tools=tools)), "acompletion")
assert sent["tools"] is tools

View file

@ -1,4 +1,5 @@
import asyncio
from types import SimpleNamespace
from typing import Final
from unittest.mock import AsyncMock, Mock
@ -19,6 +20,21 @@ from litellm.types.mcp import MCPTransport
from litellm.types.mcp_server.mcp_server_manager import MCPServer
def allow_one_request(monkeypatch: pytest.MonkeyPatch, user_id: str, server_id: str) -> None:
from litellm.proxy import proxy_server
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
windows: Final = {(user_id, server_id): asyncio.Semaphore(1)}
async def enforce(user: UserAPIKeyAuth | None, server: MCPServer) -> None:
window: Final = windows[(user.user_id if user is not None else None, server.server_id)]
if window.locked():
raise ProxyRateLimitError(detail=f"{server_id} RPM exceeded")
await window.acquire()
monkeypatch.setattr(proxy_server, "proxy_logging_obj", SimpleNamespace(enforce_mcp_server_rate_limits=enforce))
def context(session: ServerSession, revision: str = "2026-07-28") -> ServerRequestContext:
return ServerRequestContext(
session=session,
@ -356,3 +372,117 @@ async def test_proxy_route_rejects_legacy_resource_subscription_before_setup(met
await getattr(server, method)(context(Mock(spec=ServerSession)), params)
finally:
server._mcp_proxy_mode.reset(token)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("client_requests", "tools_list_admitted"),
[((), True), (("subscriptions/listen",), False), (("resources/subscribe",), False)],
)
async def test_only_client_initiated_listeners_consume_server_rate_limit(
monkeypatch: pytest.MonkeyPatch, client_requests: tuple[str, ...], tools_list_admitted: bool
) -> None:
from litellm.proxy._experimental.mcp_server import operations
allow_one_request(monkeypatch, "alice", "upstream")
opened: Final = asyncio.Queue[None]()
async def hold_upstream(*_: object, **__: object) -> None:
opened.put_nowait(None)
await asyncio.Event().wait()
client: Final = Mock(spec=MCPClient)
client.run_with_session = AsyncMock(side_effect=hold_upstream)
server: Final = MCPServer(server_id="upstream", name="upstream", url="http://localhost/mcp", transport="http")
target: Final = subscriptions.SubscriptionTarget(server, client, "synthetic-fingerprint")
async def resolve(_: OperationContext) -> tuple[subscriptions.SubscriptionTarget, ...]:
return (target,)
service: Final = subscriptions.GatewaySubscriptions(resolve_targets=resolve)
caller: Final = OperationContext(_caller=UserAPIKeyAuth(user_id="alice"))
authorize: Final = AsyncMock(return_value=caller)
session: Final = Mock(spec=ServerSession)
session.client_params = Mock()
legacy: Final = context(session, "2025-11-25")
requests: Final = {
"subscriptions/listen": lambda: service.listen(
context(Mock(spec=ServerSession)),
SubscriptionsListenRequestParams(notifications=SubscriptionFilter(tools_list_changed=True)),
authorize,
"alice",
),
"resources/subscribe": lambda: service.legacy_resource(legacy, "test://value", subscribe=True),
}
authorization: Final = subscriptions.SubscriptionAuthorization(caller, authorize, lambda *_: "alice")
starts: Final = (lambda: service.legacy_session(legacy, authorization), *(requests[r] for r in client_requests))
tasks: Final = tuple(asyncio.create_task(start()) for start in starts)
try:
for _ in tasks:
await asyncio.wait_for(opened.get(), 1)
rejection: Final = await operations._mcp_server_rate_limit_rejection(server, caller.user_api_key_auth)
assert (rejection is None) is tools_list_admitted
finally:
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
@pytest.mark.asyncio
async def test_legacy_listener_catalog_refreshes_do_not_consume_server_rate_limit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from mcp.types import ListToolsResult, ServerCapabilities, Tool, ToolsCapability
from litellm.proxy import proxy_server
from litellm.proxy._experimental.mcp_server import catalog, operations
from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListOk
server: Final = MCPServer(server_id="list-changed", name="list-changed", transport=MCPTransport.http)
caller: Final = UserAPIKeyAuth(api_key="list-changed-key", user_id="alice")
allow_one_request(monkeypatch, "alice", "list-changed")
listed: Final = asyncio.Queue[None]()
async def fetch_tools(_: MCPServer, **__: object) -> tuple[ListToolsResult, ServerListOk]:
listed.put_nowait(None)
return ListToolsResult(tools=[Tool(name="add", inputSchema={"type": "object"})]), ServerListOk(tool_count=1)
monkeypatch.setenv("LITELLM_SALT_KEY", "list-changed-test-key")
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(operations.global_mcp_server_manager, "registry", {server.server_id: server})
monkeypatch.setattr(operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server]))
monkeypatch.setattr(catalog, "get_filtered_server_tools", fetch_tools)
upstream: Final = Mock()
upstream.server_capabilities = ServerCapabilities(tools=ToolsCapability(list_changed=True))
upstream.protocol_version = "2025-11-25"
upstream.send_ping = AsyncMock()
async def run_with_session(connected, **_: object) -> None:
await connected(upstream)
client: Final = Mock(spec=MCPClient)
client.run_with_session = AsyncMock(side_effect=run_with_session)
target: Final = subscriptions.SubscriptionTarget(server, client, "synthetic-fingerprint")
async def resolve(_: OperationContext) -> tuple[subscriptions.SubscriptionTarget, ...]:
return (target,)
initial: Final = operations.prepare_context(user_api_key_auth=caller, mcp_servers=[server.server_id])
authorize: Final = AsyncMock(return_value=initial)
session: Final = Mock(spec=ServerSession)
session.client_params = Mock()
service: Final = subscriptions.GatewaySubscriptions(resolve_targets=resolve)
listener: Final = asyncio.create_task(
service.legacy_session(
context(session, "2025-11-25"),
subscriptions.SubscriptionAuthorization(initial, authorize, lambda *_: "alice"),
)
)
try:
for _ in range(2):
await asyncio.wait_for(listed.get(), 5)
assert await operations._mcp_server_rate_limit_rejection(server, caller) is None
finally:
listener.cancel()
await asyncio.gather(listener, return_exceptions=True)

View file

@ -5,13 +5,16 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objec
import json
import logging
import unittest
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from litellm import Router
from litellm._logging import verbose_proxy_logger
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.types.utils import TokenCountResponse
class TestAnthropicEndpoints(unittest.TestCase):
@ -467,6 +470,151 @@ class TestErrorLogCarriesCallId:
assert call_id in record.getMessage()
def _claude_router() -> Router:
return Router(
model_list=[
{
"model_name": "claude-sonnet",
"litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_key": "sk-test"},
}
]
)
_MALFORMED_TOOL_ENTRY: Final = [{"name": "get_weather", "input_schema": "object"}]
_SEVEN_TOKENS: Final = TokenCountResponse(
total_tokens=7, request_model="claude-sonnet", model_used="claude-sonnet", tokenizer_type="test"
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model", "tools"),
[
("claude-sonnet", 5),
("claude-sonnet", ""),
("claude-sonnet", ["get_weather"]),
("claude-sonnet", _MALFORMED_TOOL_ENTRY),
("team-public-name", 5),
],
ids=["int", "empty_string", "list_of_strings", "input_schema_string", "unnamed_model_int"],
)
async def test_count_tokens_answers_400_for_malformed_tools_without_counting(model: str, tools: object) -> None:
import litellm.proxy.anthropic_endpoints.endpoints as ep
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy._types import UserAPIKeyAuth
request: Final = MagicMock()
request.headers = {}
counter: Final = AsyncMock()
body: Final = {"model": model, "messages": [{"role": "user", "content": "hi"}], "tools": tools}
with (
patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts it is never reached
patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route
):
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
assert response.status_code == 400
assert json.loads(response.body)["error"]["type"] == "invalid_request_error"
counter.assert_not_awaited()
@pytest.mark.asyncio
async def test_count_tokens_counts_malformed_tool_entries_for_a_model_the_router_cannot_name() -> None:
import litellm.proxy.anthropic_endpoints.endpoints as ep
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy._types import UserAPIKeyAuth
request: Final = MagicMock()
request.headers = {}
counter: Final = AsyncMock(return_value=_SEVEN_TOKENS)
body: Final = {"model": "team-public-name", "messages": [{"role": "user", "content": "hi"}], "tools": _MALFORMED_TOOL_ENTRY}
with (
patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives
patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route
):
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
assert response == {"input_tokens": 7}
assert counter.await_args.kwargs["request"].tools == _MALFORMED_TOOL_ENTRY
@pytest.mark.asyncio
@pytest.mark.parametrize(
("dropping", "headers"),
[
({"drop_params": True}, {}),
({"additional_drop_params": ["tools"]}, {}),
({}, {"user-agent": "claude-cli/2.1.0 (external, cli)"}),
],
ids=["drop_params", "additional_drop_params", "claude_code_user_agent"],
)
async def test_count_tokens_drops_malformed_tools_when_drop_params_is_set(
dropping: dict[str, object], headers: dict[str, str]
) -> None:
import litellm.proxy.anthropic_endpoints.endpoints as ep
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy._types import UserAPIKeyAuth
request: Final = MagicMock()
request.headers = headers
counter: Final = AsyncMock(return_value=_SEVEN_TOKENS)
messages: Final = [{"role": "user", "content": "hi"}]
body: Final = {"model": "claude-sonnet", "messages": messages, "tools": 5, **dropping}
with (
patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives
):
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
assert response == {"input_tokens": 7}
counted: Final = counter.await_args.kwargs["request"]
assert counted.tools is None
assert counted.messages == messages
@pytest.mark.asyncio
async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_even_with_fallbacks() -> None:
import litellm.proxy.anthropic_endpoints.endpoints as ep
import litellm.proxy.proxy_server as proxy_server
from litellm import Router
from litellm.proxy._types import UserAPIKeyAuth
router: Final = Router(
model_list=[
{
"model_name": "claude-sonnet",
"litellm_params": {
"model": "anthropic/claude-sonnet-5-5",
"api_key": "sk-test",
"additional_drop_params": ["tools"],
},
},
{"model_name": "backup", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "sk-test"}},
],
fallbacks=[{"claude-sonnet": ["backup"]}],
)
request: Final = MagicMock()
request.headers = {}
counter: Final = AsyncMock(return_value=_SEVEN_TOKENS)
body: Final = {"model": "claude-sonnet", "messages": [{"role": "user", "content": "hi"}], "tools": 5}
with (
patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives
patch.object(proxy_server, "llm_router", new=router), # test-quality-ok: module global imported at call time; no injection seam on this route
):
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
assert response == {"input_tokens": 7}
assert counter.await_args.kwargs["request"].tools is None
class TestEventLoggingBatchEndpoint:
"""Test the stubbed event logging batch endpoint"""

View file

@ -5,6 +5,8 @@ This test file tests the Generic Guardrail API implementation,
specifically focusing on metadata extraction and passing.
"""
import json
import logging
import os
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
@ -16,6 +18,7 @@ import litellm
from litellm import ModelResponse
from litellm._version import version as litellm_version
from litellm.exceptions import GuardrailRaisedException, Timeout
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, MaskedHTTPStatusError
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
GenericGuardrailAPI,
@ -38,6 +41,38 @@ def generic_guardrail():
)
@pytest.fixture
def raw_rows_guardrail():
return GenericGuardrailAPI(
api_base="https://api.test.guardrail.com",
headers={"Authorization": "Bearer test-key"},
guardrail_name="test-generic-guardrail-raw-rows",
event_hook="pre_call",
default_on=True,
forward_messages_as_sent=True,
)
async def _structured_messages_sent(guardrail: GenericGuardrailAPI, rows: list[dict]) -> list:
sent_bodies: Final[list[bytes]] = [] # mutable-ok: records every body the guardrail endpoint received
async def answer(request: httpx.Request) -> httpx.Response:
sent_bodies.append(request.content)
return httpx.Response(200, json={"action": "NONE"}, request=request)
endpoint: Final = AsyncHTTPHandler(transport=httpx.MockTransport(answer))
try:
with patch.object(guardrail, "async_handler", endpoint):
await guardrail.apply_guardrail(
inputs={"texts": ["Look up 123-45-6789 for me."], "structured_messages": rows},
request_data={},
input_type="request",
)
finally:
await endpoint.close()
return [json.loads(body)["structured_messages"] for body in sent_bodies]
@pytest.fixture
def mock_user_api_key_dict():
"""Create a mock UserAPIKeyAuth object"""
@ -638,6 +673,157 @@ class TestStructuredMessagesInResponse:
assert returned_rows[1] is tool_call_row
assert returned_rows[2] == {"role": "tool", "tool_call_id": "call_1", "content": '{"ssn": "[REDACTED]"}'}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("row", "openai_row"),
[
(
{"role": "user", "content": "Look up 123-45-6789 for me.", "name": "pat"},
{"role": "user", "content": "Look up 123-45-6789 for me."},
),
(
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "lookup", "arguments": "{}"},
"index": 0,
}
],
},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}
],
},
),
(
{
"role": "user",
"content": [
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAA="}}
],
},
{"role": "user", "content": []},
),
],
ids=["extra_key", "tool_call_index", "image_block"],
)
async def test_rows_reach_the_guardrail_as_openai_chat_rows_by_default(
self, generic_guardrail: GenericGuardrailAPI, row, openai_row
):
assert await _structured_messages_sent(generic_guardrail, [row]) == [[openai_row]]
@pytest.mark.asyncio
async def test_a_returned_row_that_differs_from_the_shown_row_is_written_back_by_default(
self, generic_guardrail, mock_request_data_input
):
sent_row = {"role": "user", "content": "my ssn is 123-45-6789", "name": "pat"}
def answer_with_the_row_as_the_client_sent_it(url, json, headers, **_kwargs):
answer = MagicMock()
answer.json.return_value = {
"action": "GUARDRAIL_INTERVENED",
"texts": ["my ssn is [SSN]"],
"structured_messages": [sent_row],
}
answer.raise_for_status = MagicMock()
return answer
with patch.object(
generic_guardrail.async_handler, "post", side_effect=answer_with_the_row_as_the_client_sent_it
):
guardrailed_inputs = await generic_guardrail.apply_guardrail(
inputs={"texts": ["my ssn is 123-45-6789"], "structured_messages": [sent_row]},
request_data=mock_request_data_input,
input_type="request",
)
assert guardrailed_inputs["structured_messages"] == [sent_row]
@pytest.mark.asyncio
async def test_rows_echoed_back_with_one_rewrite_write_back_the_rewrite_only(
self, raw_rows_guardrail, mock_request_data_input
):
tool_call_row = {
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}, "index": 0}
],
}
original_rows = [
{"role": "user", "content": "Look up 123-45-6789 for me.", "name": "pat"},
tool_call_row,
{"role": "tool", "tool_call_id": "call_1", "content": '{"ssn": "123-45-6789"}'},
]
def echo_with_tool_output_redacted(url, json, headers, **_kwargs):
shown_rows = json["structured_messages"]
answer = MagicMock()
answer.json.return_value = {
"action": "GUARDRAIL_INTERVENED",
"texts": ["Look up 123-45-6789 for me."],
"structured_messages": [
shown_rows[0],
shown_rows[1],
{**shown_rows[2], "content": '{"ssn": "[REDACTED]"}'},
],
}
answer.raise_for_status = MagicMock()
return answer
with patch.object(raw_rows_guardrail.async_handler, "post", side_effect=echo_with_tool_output_redacted):
guardrailed_inputs = await raw_rows_guardrail.apply_guardrail(
inputs={"texts": ["Look up 123-45-6789 for me."], "structured_messages": original_rows},
request_data=mock_request_data_input,
input_type="request",
)
returned_rows = guardrailed_inputs["structured_messages"]
assert returned_rows[0] == original_rows[0]
assert returned_rows[1] == tool_call_row
assert returned_rows[2] == {"role": "tool", "tool_call_id": "call_1", "content": '{"ssn": "[REDACTED]"}'}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"row",
[
{"role": "user", "content": "Look up 123-45-6789 for me.", "name": "pat"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}, "index": 0}
],
},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "123-45-6789"}]},
{"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "lookup", "input": {}}]},
{
"role": "user",
"content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAA="}}],
},
{
"role": "user",
"content": [
{"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "ssn"}}
],
},
],
ids=["extra_key", "tool_call_index", "tool_result_block", "tool_use_block", "image_block", "document_block"],
)
async def test_rows_reach_the_guardrail_verbatim_when_forwarded_as_sent(
self, raw_rows_guardrail: GenericGuardrailAPI, row
):
"""Rows are the provider's own shape, so the guardrail must see every key and
content block as sent, not a copy filtered through the chat completion types."""
assert await _structured_messages_sent(raw_rows_guardrail, [row]) == [[row]]
@pytest.mark.asyncio
async def test_rows_all_echoed_back_as_shown_leave_the_rewrite_to_texts(
self, generic_guardrail, mock_request_data_input
@ -672,6 +858,126 @@ class TestStructuredMessagesInResponse:
assert "structured_messages" not in guardrailed_inputs
assert guardrailed_inputs["texts"] == ["Never repeat an SSN.", "Look up [REDACTED] for me."]
@pytest.mark.asyncio
async def test_rows_echoed_without_the_keys_the_server_does_not_know_leave_the_rewrite_to_texts(
self, raw_rows_guardrail, mock_request_data_input
):
"""A server that parses rows with its own message types echoes them back without
the keys those types lack, and its texts redaction must still reach the LLM."""
original_rows = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}, "index": 0}
],
"refusal": None,
"annotations": [],
},
{"role": "tool", "tool_call_id": "call_1", "content": "found"},
{"role": "user", "content": "my ssn is 123-45-6789"},
]
def echo_known_keys_and_redact_texts(url, json, headers, **_kwargs):
rows = json["structured_messages"]
answer = MagicMock()
answer.json.return_value = {
"action": "GUARDRAIL_INTERVENED",
"texts": [text.replace("123-45-6789", "[SSN]") for text in json["texts"]],
"structured_messages": [
{
"role": "assistant",
"content": None,
"refusal": None,
"tool_calls": [
{key: value for key, value in rows[0]["tool_calls"][0].items() if key != "index"}
],
},
rows[1],
rows[2],
],
}
answer.raise_for_status = MagicMock()
return answer
with patch.object(raw_rows_guardrail.async_handler, "post", side_effect=echo_known_keys_and_redact_texts):
guardrailed_inputs = await raw_rows_guardrail.apply_guardrail(
inputs={"texts": ["found", "my ssn is 123-45-6789"], "structured_messages": original_rows},
request_data=mock_request_data_input,
input_type="request",
)
assert "structured_messages" not in guardrailed_inputs
assert guardrailed_inputs["texts"] == ["found", "my ssn is [SSN]"]
@pytest.mark.asyncio
async def test_a_row_rewrite_echoes_the_other_rows_back_as_their_original_rows(
self, raw_rows_guardrail, mock_request_data_input
):
assistant_row = {
"role": "assistant",
"content": "Looking it up.",
"annotations": [{"type": "url_citation", "url_citation": {"url": "https://example.com"}}],
}
original_rows = [assistant_row, {"role": "user", "content": "my ssn is 123-45-6789"}]
redacted_row = {"role": "user", "content": "my ssn is [SSN]"}
def redact_one_row(url, json, headers, **_kwargs):
answer = MagicMock()
answer.json.return_value = {
"action": "GUARDRAIL_INTERVENED",
"structured_messages": [{"role": "assistant", "content": "Looking it up."}, redacted_row],
}
answer.raise_for_status = MagicMock()
return answer
with patch.object(raw_rows_guardrail.async_handler, "post", side_effect=redact_one_row):
guardrailed_inputs = await raw_rows_guardrail.apply_guardrail(
inputs={"texts": ["my ssn is 123-45-6789"], "structured_messages": original_rows},
request_data=mock_request_data_input,
input_type="request",
)
assert guardrailed_inputs["structured_messages"] == [assistant_row, redacted_row]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"returned_row",
[
{"role": "assistant", "content": None},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
"annotations": [],
"refusal": "blocked",
},
],
ids=["drops_a_declared_key", "changes_a_value"],
)
async def test_a_row_that_loses_a_declared_key_or_changes_a_value_is_a_rewrite(
self, raw_rows_guardrail, mock_request_data_input, returned_row
):
original_row = {
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
"annotations": [],
"refusal": None,
}
answer = MagicMock()
answer.json.return_value = {"action": "GUARDRAIL_INTERVENED", "structured_messages": [returned_row]}
answer.raise_for_status = MagicMock()
with patch.object(raw_rows_guardrail.async_handler, "post", return_value=answer):
guardrailed_inputs = await raw_rows_guardrail.apply_guardrail(
inputs={"texts": [], "structured_messages": [original_row]},
request_data=mock_request_data_input,
input_type="request",
)
assert guardrailed_inputs["structured_messages"] == [returned_row]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"structured_messages",
@ -1253,6 +1559,40 @@ class TestGenericGuardrailAPIStreamingConfig:
)
assert guardrail.streaming_transform_mode == "incremental_diff"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("configured", "forwards_as_sent"),
[
pytest.param({}, False, id="unset"),
pytest.param({"forward_messages_as_sent": ""}, False, id="blank"),
pytest.param({"forward_messages_as_sent": False}, False, id="false"),
pytest.param({"forward_messages_as_sent": "true"}, True, id="string-true"),
pytest.param({"forward_messages_as_sent": True}, True, id="true"),
],
)
async def test_forward_messages_as_sent_config_decides_which_rows_reach_the_guardrail(
self, configured, forwards_as_sent
):
from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
initialize_guardrail,
)
from litellm.types.guardrails import LitellmParams
litellm_params = LitellmParams.model_validate(
{
"guardrail": "generic_guardrail_api",
"mode": "pre_call",
"api_base": "https://api.test.guardrail.com",
**configured,
}
)
with patch("litellm.logging_callback_manager.add_litellm_callback"):
guardrail = initialize_guardrail(litellm_params, {"guardrail_name": "test-forward-as-sent"})
row = {"role": "user", "content": "hi", "name": "pat"}
expected_row = row if forwards_as_sent else {"role": "user", "content": "hi"}
assert await _structured_messages_sent(guardrail, [row]) == [[expected_row]]
def test_initialize_guardrail_forwards_streaming_flags(self):
from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
initialize_guardrail,
@ -1945,6 +2285,34 @@ class TestToolSupport:
assert forwarded_tools == tools
@pytest.mark.asyncio
async def test_provider_native_tools_without_a_type_reach_the_guardrail_and_come_back_unchanged(
self, generic_guardrail
):
"""Gemini tools are declared as function_declarations, googleSearch and the
like with no type key, so they must be scanned, and a server that echoes
them back must not write a type key into the request to the provider."""
tools = [
{"function_declarations": [{"name": "get_weather", "parameters": {"type": "object", "properties": {}}}]},
{"googleSearch": {}},
]
def echo_tools(url, json, headers, **_kwargs):
answer = MagicMock()
answer.json.return_value = {"action": "NONE", "texts": ["hi"], "tools": json["tools"]}
answer.raise_for_status = MagicMock()
return answer
with patch.object(generic_guardrail.async_handler, "post", side_effect=echo_tools) as mock_post:
guardrailed_inputs = await generic_guardrail.apply_guardrail(
inputs={"texts": ["hi"], "tools": tools},
request_data={},
input_type="request",
)
assert mock_post.call_args.kwargs["json"]["tools"] == tools
assert guardrailed_inputs["tools"] == tools
class TestFailOnError:
"""Test fail_on_error: complete fail-open on any guardrail error"""
@ -2032,6 +2400,59 @@ class TestFailOnError:
assert result == {"texts": ["model output"]}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("error", "error_type", "pre_pr_text"),
[
(
Timeout(message="guardrail took too long", model="gpt-5", llm_provider="openai"),
"Timeout",
"Proceeding without guardrail. guardrail_name=",
),
(
httpx.ConnectError("connection refused", request=MagicMock()),
"ConnectError",
"Proceeding without guardrail. guardrail_name=",
),
(
MaskedHTTPStatusError(
httpx.HTTPStatusError(
"bad gateway",
request=httpx.Request("POST", "http://guardrail.invalid/scan"),
response=httpx.Response(502, request=httpx.Request("POST", "http://guardrail.invalid/scan")),
)
),
"HTTPStatusError",
"Proceeding without guardrail. http_status_code=502 guardrail_name=",
),
],
ids=["timeout", "connect_error", "http_status"],
)
async def test_fail_open_log_line_names_the_error_type(
self, fail_open_guardrail, caplog: pytest.LogCaptureFixture, error, error_type, pre_pr_text
):
"""An operator grepping the fail-open line must be able to tell a timeout
from a refused connection from an upstream status without the traceback."""
from litellm._logging import verbose_proxy_logger
verbose_proxy_logger.addHandler(caplog.handler)
try:
with patch.object(fail_open_guardrail.async_handler, "post", side_effect=error):
await fail_open_guardrail.apply_guardrail(
inputs={"texts": ["hi"]},
request_data={},
input_type="request",
)
finally:
verbose_proxy_logger.removeHandler(caplog.handler)
fail_open_lines: Final = [
record.getMessage() for record in caplog.records if record.levelno == logging.CRITICAL
]
assert fail_open_lines
assert all(f"error_type={error_type}" in line.split() for line in fail_open_lines)
assert all(pre_pr_text in line for line in fail_open_lines)
@pytest.mark.asyncio
async def test_response_path_valid_block_still_blocks(self, fail_open_guardrail):
"""On the response path too, a valid BLOCKED decision raises despite fail_on_error=False."""

View file

@ -953,6 +953,17 @@ async def test_provider_specific_params_includes_embedding_toggle():
assert field["default_value"] is False
@pytest.mark.asyncio
async def test_provider_specific_params_offers_forward_messages_as_sent_off_by_default():
from litellm.proxy.guardrails.guardrail_endpoints import get_provider_specific_params
provider_params = await get_provider_specific_params()
field = provider_params["generic_guardrail_api"]["optional_params"]["fields"]["forward_messages_as_sent"]
assert field["type"] == "boolean"
assert field["default_value"] is False
@pytest.mark.asyncio
async def test_provider_specific_params_exposes_bedrock_streaming_flags():
from litellm.proxy.guardrails.guardrail_endpoints import get_provider_specific_params

View file

@ -201,6 +201,10 @@ class RecordingObserver(PromptCacheObserver):
await super().async_log_success_event(kwargs, response_obj, start_time, end_time)
self.finished.set()
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
await super().async_log_failure_event(kwargs, response_obj, start_time, end_time)
self.finished.set()
def native_response():
return {

View file

@ -2,7 +2,7 @@ import asyncio
import json
import logging
import os
from collections.abc import Awaitable, Callable, Mapping
from collections.abc import Awaitable, Callable, Mapping, Sequence
from contextlib import ExitStack, asynccontextmanager
from types import SimpleNamespace
from typing import Final
@ -17,7 +17,7 @@ from fastapi import HTTPException, Request
import litellm
from litellm._uuid import uuid
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy._types import LiteLLM_UserTable, NewUserResponse
from litellm.proxy._types import LiteLLM_UserTable, NewUserResponse, ProxyException
from litellm.proxy.auth.handle_jwt import JWTHandler
from litellm.proxy.management_endpoints.sso import CustomMicrosoftSSO
from litellm.proxy.management_endpoints.types import CustomOpenID
@ -26,9 +26,11 @@ from litellm.proxy.management_endpoints.ui_sso import (
MicrosoftSSOHandler,
SSOAuthenticationHandler,
_complete_cli_sso_callback_session,
_insert_sso_user_or_find_concurrent_insert,
_setup_team_mappings,
_sync_user_role_from_jwt_role_map,
cli_sso_callback,
insert_sso_user,
normalize_email,
process_sso_jwt_access_token,
sso_lookup_user_ids,
@ -1425,6 +1427,82 @@ async def test_upsert_sso_user_no_role_in_sso_response():
assert "user_role" not in call_args.kwargs["data"]
class _UserTable:
def __init__(self, *rows: LiteLLM_UserTable) -> None:
self.rows: dict[str, LiteLLM_UserTable] = {row.user_id: row for row in rows}
async def find_first(self, where: dict[str, dict[str, str]]) -> LiteLLM_UserTable | None:
((field, condition),) = where.items()
def matches(stored: object) -> bool:
if condition.get("mode") == "insensitive":
return str(stored).lower() == condition["equals"].lower()
return stored == condition["equals"]
return next((row for row in self.rows.values() if matches(getattr(row, field))), None)
async def find_unique(self, where: dict[str, str]) -> LiteLLM_UserTable | None:
return self.rows.get(where["user_id"])
def _users_db(table: _UserTable) -> SimpleNamespace:
database: Final = SimpleNamespace(litellm_usertable=table)
return SimpleNamespace(db=database, writer_db=database)
async def _first_sign_in(table: _UserTable, **join: object) -> NewUserResponse | LiteLLM_UserTable:
prisma: Final = _users_db(table)
with patch("litellm.proxy.proxy_server.prisma_client", prisma):
return await _insert_sso_user_or_find_concurrent_insert(
insert=insert_sso_user(
result_openid=CustomOpenID(id="sso-subject", email="person@example.com", team_ids=[]),
user_defined_values={
"user_id": "sso-subject",
"user_email": "person@example.com",
"user_role": None,
"max_budget": None,
"budget_duration": None,
"models": [],
},
),
user_id="sso-subject",
prisma_client=prisma,
**join,
)
@pytest.mark.asyncio
async def test_first_sign_in_signs_in_to_the_account_a_concurrent_first_sign_in_created():
winner: Final = LiteLLM_UserTable(user_id="sso-subject", user_email="person@example.com", user_role="internal_user")
signed_in: Final = await _first_sign_in(_UserTable(winner))
assert signed_in == winner
@pytest.mark.asyncio
async def test_first_sign_in_does_not_sign_in_to_another_account_holding_the_email():
other: Final = LiteLLM_UserTable(user_id="someone-else", user_email="Person@example.com", user_role="proxy_admin")
with pytest.raises(ProxyException, match=r"User with email Person@example\.com already exists"):
await _first_sign_in(_UserTable(other))
@pytest.mark.asyncio
async def test_first_sign_in_joins_default_teams_the_concurrent_first_sign_in_has_not_joined_yet(monkeypatch):
monkeypatch.setattr(litellm, "default_internal_user_params", {"teams": ["team-joined", "team-pending"]})
table: Final = _UserTable(
LiteLLM_UserTable(user_id="sso-subject", user_email="person@example.com", teams=["team-joined"])
)
async def join(user: LiteLLM_UserTable, teams: Sequence[object], prisma_client: object) -> None:
table.rows[user.user_id] = user.model_copy(update={"teams": [*user.teams, *map(str, teams)]})
signed_in: Final = await _first_sign_in(table, join_default_teams=join)
assert signed_in.teams == ["team-joined", "team-pending"]
def test_get_user_email_and_id_extracts_microsoft_role():
"""
Test that _get_user_email_and_id_from_result extracts user_role from Microsoft SSO.

View file

@ -5,11 +5,10 @@ from typing import Any, Dict, Final, List
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import httpx
import litellm
import pytest
from pydantic import ValidationError
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.pass_through.stream_assembly import extract_sse_data
@ -2644,6 +2643,58 @@ class TestRecordPartialUsageForFailure:
assert "response_cost" not in logging_obj.model_call_details
class _FailureRecorder(CustomLogger):
def __init__(self):
super().__init__()
self.failure_kwargs = []
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
self.failure_kwargs.append(kwargs)
class TestLogErrorFrameAsFailure:
"""A stream closed at the provider's `event: error` frame is one failed attempt: its failure log bills the
usage streamed before the frame on the attempt-scoped copy its caller hands it, and the request's shared
logging state stays as it was, so the attempt the router opens next logs with nothing left behind."""
@pytest.mark.asyncio
async def test_logs_a_failure_with_the_usage_streamed_before_the_frame_and_leaves_the_request_state_alone(self):
recorder = _FailureRecorder()
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hello"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="test-error-frame-failure",
function_id="test-error-frame-failure",
dynamic_async_failure_callbacks=[recorder],
)
chunks = [
*TestRecordPartialUsageForFailure()._interrupted_chunks()[:1],
TestRecordPartialUsageForFailure._sse(
"error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}
),
]
await AnthropicPassthroughLoggingHandler.log_error_frame_as_failure(
attempt_logging_obj=logging_obj.attempt_scoped_copy(),
request_body={"model": "claude-sonnet-5", "stream": True},
all_chunks=chunks,
error_event=("overloaded_error", "Overloaded", 529),
)
assert len(recorder.failure_kwargs) == 1
failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
assert failure_payload["status"] == "failure"
assert failure_payload["prompt_tokens"] == 52
assert failure_payload["response_cost"] > 0
assert isinstance(recorder.failure_kwargs[0]["exception"], litellm.InternalServerError)
assert "combined_usage_object" not in logging_obj.model_call_details
assert "response_cost" not in logging_obj.model_call_details
assert logging_obj.should_run_logging("async_failure")
@pytest.mark.parametrize(
("all_chunks", "interrupted"),
[

View file

@ -18,6 +18,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthr
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import aclose_superseded_attempt
from litellm.proxy.pass_through_endpoints.streaming_handler import (
PassThroughStreamingHandler,
)
@ -749,6 +750,294 @@ async def test_chunk_processor_logs_failure_not_success_on_mid_stream_exception(
assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout)
def _anthropic_message_start_frame(message_id: str = "msg_1", input_tokens: int = 52) -> bytes:
return _anthropic_sse(
"message_start",
{
"type": "message_start",
"message": {
"id": message_id,
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5",
"content": [],
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": input_tokens, "output_tokens": 1},
},
},
)
def _anthropic_completed_stream_tail() -> tuple[bytes, ...]:
return (
_anthropic_sse(
"content_block_start",
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
),
_anthropic_sse(
"content_block_delta",
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}},
),
_anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}),
_anthropic_sse(
"message_delta",
{"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 3}},
),
_anthropic_sse("message_stop", {"type": "message_stop"}),
)
async def _settle(recorded: list[dict[str, object]]) -> None:
for _ in range(300):
if recorded:
return
await asyncio.sleep(0.01)
def _anthropic_overloaded_error_frame() -> bytes:
return _anthropic_sse("error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}})
def _anthropic_stream_of(frames: tuple[bytes, ...]):
mock = MagicMock(spec=httpx.Response)
mock.status_code = 200
async def _aiter_bytes():
for frame in frames:
yield frame
mock.aiter_bytes = _aiter_bytes
return mock
@pytest.mark.parametrize(
"frames, billed",
[
pytest.param(
(_anthropic_message_start_frame(), _anthropic_overloaded_error_frame()), True, id="error-after-message-start"
),
pytest.param((_anthropic_overloaded_error_frame(),), False, id="error-frame-only"),
],
)
@pytest.mark.asyncio
async def test_chunk_processor_logs_a_provider_error_frame_as_a_failure_when_closed_at_the_frame(frames, billed):
"""The router retries a /v1/messages stream whose provider sent `event: error` before any content by closing
it at the frame. That attempt never completed: it logs once as a failure carrying whatever usage the provider
had reported, and the request's one-shot success log stays free for the retry that shares the logging object."""
recorder = _EventRecorder()
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="test-error-frame",
function_id="test-error-frame",
dynamic_async_success_callbacks=[recorder],
dynamic_async_failure_callbacks=[recorder],
)
stream = PassThroughStreamingHandler.chunk_processor(
response=_anthropic_stream_of(frames),
request_body={"model": "claude-sonnet-5", "stream": True},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
)
received = []
async for chunk in stream:
received.append(chunk)
if b"event: error" in chunk:
break
await aclose_superseded_attempt(stream)
await _settle(recorder.failure_kwargs)
assert received == list(frames)
assert recorder.success_kwargs == []
assert logging_obj.model_call_details.get("has_dispatched_final_stream_success") is not True
assert "combined_usage_object" not in logging_obj.model_call_details
assert "response_cost" not in logging_obj.model_call_details
assert logging_obj.should_run_logging("async_failure")
assert len(recorder.failure_kwargs) == 1
failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
assert failure_payload["status"] == "failure"
assert isinstance(recorder.failure_kwargs[0]["exception"], litellm.InternalServerError)
if billed:
assert failure_payload["prompt_tokens"] == 52
assert failure_payload["response_cost"] > 0
else:
assert failure_payload["response_cost"] == 0
@pytest.mark.asyncio
async def test_chunk_processor_logs_a_stream_that_ended_at_the_provider_error_frame_through_the_success_route():
"""A provider error frame that ends a stream nobody closed early is the request's final answer, and that
stream keeps logging the way it always has: once, through the success route, with the usage it carried."""
recorder = _EventRecorder()
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="test-final-error-frame",
function_id="test-final-error-frame",
dynamic_async_success_callbacks=[recorder],
dynamic_async_failure_callbacks=[recorder],
)
frames = (_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())
received = [
chunk
async for chunk in PassThroughStreamingHandler.chunk_processor(
response=_anthropic_stream_of(frames),
request_body={"model": "claude-sonnet-5", "stream": True},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
)
]
await _settle(recorder.success_kwargs)
assert received == list(frames)
assert recorder.failure_kwargs == []
success_payload = recorder.success_kwargs[0]["standard_logging_object"]
assert success_payload["status"] == "success"
assert success_payload["prompt_tokens"] == 52
assert logging_obj.model_call_details["prompt_cache_response_complete"] is False
@pytest.mark.asyncio
async def test_chunk_processor_bills_a_stream_the_client_closed_at_the_provider_error_frame():
"""The Anthropic SDK raises on a provider `event: error` frame and drops the connection, so a non-retried
stream is closed at the frame by the client, not by a retry taking over. Its usage must still reach the
success route, or the tokens the provider billed never reach the key's spend."""
recorder = _EventRecorder()
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="test-client-closed-at-error-frame",
function_id="test-client-closed-at-error-frame",
dynamic_async_success_callbacks=[recorder],
dynamic_async_failure_callbacks=[recorder],
)
stream = PassThroughStreamingHandler.chunk_processor(
response=_anthropic_stream_of((_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())),
request_body={"model": "claude-sonnet-5", "stream": True},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
)
async for chunk in stream:
if b"event: error" in chunk:
break
await stream.aclose()
await _settle(recorder.success_kwargs)
assert recorder.failure_kwargs == []
success_payload = recorder.success_kwargs[0]["standard_logging_object"]
assert success_payload["status"] == "success"
assert success_payload["prompt_tokens"] == 52
@pytest.mark.asyncio
async def test_superseded_attempt_failure_log_keeps_its_own_deployment_when_the_retry_rebinds_the_logging_object():
"""The router rebinds the shared logging object to the retry's deployment right after closing the superseded
stream, before the logging worker runs. The failure log of the superseded attempt must still name the
deployment that served it, or the router's cooldown lands on the deployment serving the retry."""
recorder = _EventRecorder()
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="test-superseded-deployment",
function_id="test-superseded-deployment",
dynamic_async_failure_callbacks=[recorder],
)
logging_obj.update_environment_variables(
model="claude-sonnet-5", litellm_params={"model_info": {"id": "deployment-1"}}, optional_params={}
)
superseded = PassThroughStreamingHandler.chunk_processor(
response=_anthropic_stream_of((_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())),
request_body={"model": "claude-sonnet-5", "stream": True},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
)
async for chunk in superseded:
if b"event: error" in chunk:
break
await aclose_superseded_attempt(superseded)
logging_obj.update_environment_variables(
model="claude-sonnet-5-retry", litellm_params={"model_info": {"id": "deployment-2"}}, optional_params={}
)
await _settle(recorder.failure_kwargs)
failure_kwargs = recorder.failure_kwargs[0]
assert failure_kwargs["litellm_params"]["model_info"]["id"] == "deployment-1"
assert failure_kwargs["model"] == "claude-sonnet-5"
assert logging_obj.model_call_details["litellm_params"]["model_info"]["id"] == "deployment-2"
@pytest.mark.asyncio
async def test_retry_success_log_carries_its_own_usage_after_a_provider_error_frame():
"""Both attempts of a retried /v1/messages stream log through the request's one logging object. The
superseded attempt's failure log must not leave the usage it consumed behind, or the retry's success
log bills the failed attempt's tokens instead of its own."""
recorder = _EventRecorder()
logging_obj = LiteLLMLoggingObj(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.now(),
litellm_call_id="test-retry-own-usage",
function_id="test-retry-own-usage",
dynamic_async_success_callbacks=[recorder],
dynamic_async_failure_callbacks=[recorder],
)
def attempt(frames: tuple[bytes, ...]):
return PassThroughStreamingHandler.chunk_processor(
response=_anthropic_stream_of(frames),
request_body={"model": "claude-sonnet-5", "stream": True},
litellm_logging_obj=logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.now(),
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
)
superseded = attempt((_anthropic_message_start_frame("msg_a1", 52), _anthropic_overloaded_error_frame()))
async for chunk in superseded:
if b"event: error" in chunk:
break
await aclose_superseded_attempt(superseded)
await _settle(recorder.failure_kwargs)
retry = attempt((_anthropic_message_start_frame("msg_a2", 7), *_anthropic_completed_stream_tail()))
async for _ in retry:
pass
await _settle(recorder.success_kwargs)
failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
assert failure_payload["prompt_tokens"] == 52
success_payload = recorder.success_kwargs[0]["standard_logging_object"]
assert success_payload["status"] == "success"
assert (success_payload["prompt_tokens"], success_payload["completion_tokens"]) == (7, 3)
def _google_sse(prompt_tokens: int, completion_tokens: int, text: str) -> bytes:
payload = {
"candidates": [{"content": {"parts": [{"text": text}], "role": "model"}, "index": 0}],

View file

@ -3454,10 +3454,11 @@ class TestResponsesInputTokens:
body: dict[str, Any],
path: str = "/v1/responses/input_tokens",
counter: AsyncMock | None = None,
llm_router: litellm.Router | None = None,
) -> tuple[Response, AsyncMock]:
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.response_api_endpoints.endpoints import _proxy_token_counter
from litellm.proxy.response_api_endpoints.endpoints import _proxy_llm_router, _proxy_token_counter
from litellm.types.utils import TokenCountResponse
token_counter_mock = (
@ -3474,6 +3475,7 @@ class TestResponsesInputTokens:
)
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test", request_route=path)
app.dependency_overrides[_proxy_token_counter] = lambda: token_counter_mock
app.dependency_overrides[_proxy_llm_router] = lambda: llm_router
try:
client = TestClient(app)
response = client.post(path, json=body, headers={"Authorization": "Bearer sk-9876"})
@ -3481,6 +3483,7 @@ class TestResponsesInputTokens:
finally:
app.dependency_overrides.pop(user_api_key_auth, None)
app.dependency_overrides.pop(_proxy_token_counter, None)
app.dependency_overrides.pop(_proxy_llm_router, None)
def test_string_input_returns_openai_input_tokens_shape(self):
response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "Hello, how are you?"})
@ -3594,6 +3597,48 @@ class TestResponsesInputTokens:
assert response.status_code == 400, response.text
error = response.json()["error"]
assert error["type"] == "invalid_request_error"
assert error["param"] == "tools"
assert error["message"].startswith("tools: ")
counter.assert_not_awaited()
def test_invalid_tools_are_dropped_under_drop_params(self) -> None:
response, counter = self._post_input_tokens(
{"model": "gpt-4o", "input": "hi", "tools": [5, {"type": "web_search"}], "drop_params": True}
)
assert response.status_code == 200, response.text
assert counter.await_args.kwargs["request"].tools == [{"type": "web_search"}]
def test_malformed_tools_follow_the_counted_deployment_even_when_fallbacks_are_configured(self) -> None:
router: Final = litellm.Router(
model_list=[
{
"model_name": "gpt-5.2",
"litellm_params": {"model": "openai/gpt-5.2", "api_key": "sk-test", "additional_drop_params": ["tools"]},
},
{"model_name": "backup", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-test"}},
],
fallbacks=[{"gpt-5.2": ["backup"]}],
)
response, counter = self._post_input_tokens(
{"model": "gpt-5.2", "input": "hi", "tools": 5}, llm_router=router
)
assert response.status_code == 200, response.text
assert counter.await_args.kwargs["request"].tools is None
def test_malformed_tools_for_a_model_outside_the_router_get_a_clean_400(self) -> None:
router: Final = litellm.Router(
model_list=[{"model_name": "gpt-5.2", "litellm_params": {"model": "openai/gpt-5.2", "api_key": "sk-test"}}]
)
response, counter = self._post_input_tokens(
{"model": "openai/gpt-5.4-mini", "input": "hi", "tools": 5}, llm_router=router
)
assert response.status_code == 400, response.text
assert response.json()["error"]["param"] == "tools"
counter.assert_not_awaited()
def test_provider_error_maps_status_code(self):

View file

@ -145,6 +145,21 @@ class TestExtractRequestToolNames:
data = {"tools": [{"type": "function", "function": {"name": "x"}}]}
assert extract_request_tool_names("/v1/embeddings", data) == []
@pytest.mark.parametrize(
("route", "data"),
[
("/v1/chat/completions", {"tools": 5}),
("/v1/chat/completions", {"functions": 5}),
("/v1/responses", {"tools": 5}),
("/v1/messages", {"tools": 5}),
("/generate_content", {"tools": 5}),
("/generate_content", {"tools": [{"functionDeclarations": 5}]}),
],
ids=["chat_tools", "chat_functions", "responses", "messages", "generate_content", "function_declarations"],
)
def test_malformed_tool_lists_yield_no_names(self, route: str, data: dict[str, object]) -> None:
assert extract_request_tool_names(route, data) == []
class TestCheckToolsAllowlist:
"""Test allowlist enforcement in auth (no DB in hot path)."""

View file

@ -4,6 +4,7 @@ and ``_handle_logging_proxy_only_error``."""
from __future__ import annotations
import asyncio
from collections.abc import Callable
from datetime import datetime
from types import MappingProxyType
from typing import Final
@ -921,6 +922,48 @@ async def test_post_call_failure_hook_falls_back_to_body_shape_without_a_route(p
assert request_data["call_type"] == "acompletion"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"call_type, body",
[
("anthropic_messages", {"model": "m", "messages": [{"role": "user", "content": "hi"}]}),
("aresponses", {"model": "m", "input": [{"role": "user", "content": "hi"}]}),
("acompact_responses", {"model": "m", "input": [{"role": "user", "content": "hi"}]}),
],
)
async def test_post_call_failure_hook_keeps_the_call_type_request_setup_logged(
proxy_logging: ProxyLogging,
make_user_api_key_auth: Callable[..., UserAPIKeyAuth],
call_type: str,
body: dict[str, object],
) -> None:
"""A request rejected after request setup already carries its endpoint's call type, so the body
shape must not relabel a /v1/messages rejection as a chat completion or a list-input /v1/responses
rejection as an embedding."""
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
proxy_logging.alert_types = []
request_data: Final = {
**body,
"litellm_logging_obj": LiteLLMLoggingObj(
model="m",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type=call_type,
start_time=datetime.now(),
litellm_call_id=f"test_keeps_{call_type}",
function_id=f"test_keeps_{call_type}",
),
"metadata": {},
}
await proxy_logging.post_call_failure_hook(
request_data=request_data,
original_exception=HTTPException(status_code=400, detail="tools must be a list"),
user_api_key_dict=make_user_api_key_auth(request_route="/v1/messages"),
)
assert request_data["call_type"] == call_type
@pytest.mark.asyncio
@pytest.mark.parametrize("route", ["/v1/files", "/files/file-abc", "/v1/containers"])
async def test_post_call_failure_hook_keeps_the_route_for_multi_operation_routes(

View file

@ -287,6 +287,19 @@ class TestFilterWebSearchDeployments:
assert "deployment-2" in result_ids
assert "deployment-3" not in result_ids
def test_web_search_in_a_tuple_filters_unsupported(self, sample_deployments: list[dict[str, object]]) -> None:
result: Final = filter_web_search_deployments(sample_deployments, {"tools": ({"type": "web_search"},)})
assert [d["model_info"]["id"] for d in result] == ["deployment-1", "deployment-2"]
@pytest.mark.parametrize("tool_type", [{}, [], None, 5], ids=["object", "list", "null", "int"])
def test_a_non_string_tool_type_is_no_web_search_request(
self, sample_deployments: list[dict[str, object]], tool_type: object
) -> None:
result: Final = filter_web_search_deployments(
sample_deployments, {"tools": [{"type": tool_type}, {"type": "function"}]}
)
assert result == sample_deployments
def test_web_search_preview_filters_unsupported(self, sample_deployments):
"""web_search_preview type should also trigger filtering"""
request_kwargs = {"tools": [{"type": "web_search_preview"}]}

View file

@ -9,6 +9,8 @@ the question neither covers: whether the job that globs a file then deselects it
"""
import importlib.util
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Final
@ -71,17 +73,27 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path:
workflow.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": "pytest tests/integration"}]}}}))
_, findings = coverage._integration_ownership(tmp_path)
assert [(finding.subject, finding.detail) for finding in findings] == [
(test_path, "integration contract is also selected by GitHub Actions")
(test_path, "integration file outside GITHUB_FILES is selected by GitHub Actions")
]
github_path: Final = "tests/integration/management/test_github_contract.py"
github_path: Final = "tests/integration/database/image/test_github_contract.py"
(tmp_path / github_path).parent.mkdir(parents=True)
(tmp_path / github_path).write_text("def test_contract(): pass\n")
runner: Final = tmp_path / "tests/integration/run.py"
runner.write_text(runner.read_text() + f"GITHUB_FILES: Final = frozenset({{{github_path!r}}})\n")
workflow.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}}))
workflow.unlink()
dedicated: Final = tmp_path / ".github/workflows/image-scan.yml"
dedicated.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}}))
github_owned, github_findings = coverage._integration_ownership(tmp_path)
assert github_owned == frozenset({test_path, github_path})
assert github_findings == ()
workflow.write_text(yaml.safe_dump({"jobs": {}}))
per_pr: Final = tmp_path / ".github/workflows/test-mcp-interactions.yml"
per_pr.write_text(yaml.safe_dump({"jobs": {"mcp": {"steps": [{"run": f"pytest {github_path}"}]}}}))
_, shared_findings = coverage._integration_ownership(tmp_path)
assert [(finding.subject, finding.detail) for finding in shared_findings] == [
(github_path, "GitHub-owned integration contract runs outside a dedicated integration workflow")
]
per_pr.unlink()
dedicated.write_text(yaml.safe_dump({"jobs": {}}))
_, missing_invocation = coverage._integration_ownership(tmp_path)
assert [(finding.subject, finding.detail) for finding in missing_invocation] == [
(github_path, "GitHub-owned integration contract has no invoking job")
@ -94,6 +106,72 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path:
assert circle_invocation == ()
def _write_integration_tree(root: Path, github_path: str) -> None:
contracts: Final = root / "tests/integration/management"
contracts.mkdir(parents=True)
(contracts / "test_contract.py").write_text("def test_contract(): pass\n")
(root / github_path).parent.mkdir(parents=True, exist_ok=True)
(root / github_path).write_text("def test_contract(): pass\n")
(root / "tests/integration/run.py").write_text(
"from types import MappingProxyType\nfrom typing import Final\n"
'GROUPS: Final = MappingProxyType({"management": ("management",)})\n'
f"GITHUB_FILES: Final = frozenset({{{github_path!r}}})\n"
)
(root / ".circleci").mkdir()
(root / ".circleci/config.yml").write_text(
yaml.safe_dump(
{
"jobs": {
"integration_contracts": {
"steps": [{"run": {"command": "bash .circleci/scripts/run_integration.sh management"}}]
}
},
"workflows": {"integration": {"jobs": [{"integration_contracts": {"suite": "management"}}]}},
}
)
)
(root / ".github/workflows").mkdir(parents=True)
@pytest.mark.parametrize(
("github_path", "workflow_name", "exit_code"),
[
("tests/integration/database/image/test_github_contract.py", "image-scan.yml", 0),
("tests/integration/database/image/test_github_contract.py", "test-unit.yml", 1),
("tests/integration/management/test_github_contract.py", "image-scan.yml", 1),
],
)
def test_census_fails_when_a_github_owned_contract_runs_outside_its_dedicated_workflow(
tmp_path: Path, github_path: str, workflow_name: str, exit_code: int
) -> None:
script: Final = tmp_path / ".github/scripts/assert_ci_coverage.py"
script.parent.mkdir(parents=True)
shutil.copy(_MODULE_PATH, script)
_write_integration_tree(tmp_path, github_path)
(tmp_path / ".github/workflows" / workflow_name).write_text(
yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}})
)
result: Final = subprocess.run([sys.executable, "-I", str(script)], capture_output=True, text=True, check=False)
assert result.returncode == exit_code, result.stdout
assert "Playwright" not in result.stdout
def test_an_integration_file_outside_every_group_cannot_run_on_a_github_shard(tmp_path: Path) -> None:
_write_integration_tree(tmp_path, "tests/integration/database/image/test_github_contract.py")
ungrouped: Final = "tests/integration/ci_probe/test_ci_probe.py"
(tmp_path / ungrouped).parent.mkdir(parents=True)
(tmp_path / ungrouped).write_text("def test_probe(): pass\n")
_, before = coverage._integration_ownership(tmp_path)
(tmp_path / ".github/workflows/test-unit.yml").write_text(
yaml.safe_dump({"jobs": {"mcp": {"steps": [{"run": f"pytest {ungrouped}"}]}}})
)
_, after = coverage._integration_ownership(tmp_path)
assert [(finding.subject, finding.detail) for finding in after if finding.subject == ungrouped] == [
(ungrouped, "integration file outside GITHUB_FILES is selected by GitHub Actions")
]
assert not [finding for finding in before if finding.subject == ungrouped]
def test_circleci_postgres_suites_count_their_test_path_as_invoked() -> None:
config: Final = {
"workflows": {

View file

@ -38,6 +38,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.plumbing.context import set_request_root_span
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import closing_superseded_attempt
from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import (
SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES,
)
@ -9739,6 +9740,96 @@ def test_update_kwargs_with_deployment_merge_tools_request_overrides_tool_choice
assert kwargs["tool_choice"] == "none"
@pytest.mark.parametrize(
"deployment_params, router_defaults, request_params, expected_tools",
[
({"drop_params": True}, {}, {}, [{"type": "web_search"}]),
({}, {"drop_params": True}, {}, [{"type": "web_search"}]),
({}, {}, {"additional_drop_params": ["tools"]}, [{"type": "web_search"}]),
({"drop_params": True}, {}, {"drop_params": False}, 5),
({}, {}, {}, 5),
],
ids=[
"deployment_drop_params",
"router_default_drop_params",
"request_drops_tools",
"request_overrides_deployment_drop",
"no_drop_policy",
],
)
def test_update_kwargs_with_deployment_keeps_deployment_tools_when_malformed_request_tools_drop(
deployment_params: dict[str, object],
router_defaults: dict[str, object],
request_params: dict[str, object],
expected_tools: object,
) -> None:
router: Final = litellm.Router(
model_list=[
{
"model_name": "o3-deep-research",
"litellm_params": {
"model": "openai/o3-deep-research",
"api_key": "fake-key",
"tools": [{"type": "web_search"}],
**deployment_params,
},
},
],
default_litellm_params={**router_defaults},
)
kwargs: Final[dict[str, object]] = {"metadata": {}, "tools": 5, **request_params}
router._update_kwargs_with_deployment(
deployment=router.get_deployment_by_model_group_name(model_group_name="o3-deep-research"), kwargs=kwargs
)
assert kwargs["tools"] == expected_tools
_NAMED_REQUEST_TOOL_CHOICE: Final = {"type": "function", "function": {"name": "lookup_order"}}
@pytest.mark.parametrize(
"deployment_params, expected_tool_fields",
[
(
{"drop_params": True, "tool_choice": "required"},
{"tools": [{"type": "web_search"}], "tool_choice": "required"},
),
({"drop_params": True}, {"tools": [{"type": "web_search"}]}),
({}, {"tools": 5, "tool_choice": _NAMED_REQUEST_TOOL_CHOICE, "parallel_tool_calls": False}),
],
ids=["deployment_tool_choice_applies", "no_deployment_tool_choice", "no_drop_policy"],
)
def test_update_kwargs_with_deployment_drops_request_tool_settings_with_malformed_request_tools(
deployment_params: dict[str, object], expected_tool_fields: dict[str, object]
) -> None:
router: Final = litellm.Router(
model_list=[
{
"model_name": "o3-deep-research",
"litellm_params": {
"model": "openai/o3-deep-research",
"api_key": "fake-key",
"tools": [{"type": "web_search"}],
**deployment_params,
},
},
],
)
kwargs: Final[dict[str, object]] = {
"metadata": {},
"tools": 5,
"tool_choice": _NAMED_REQUEST_TOOL_CHOICE,
"parallel_tool_calls": False,
}
router._update_kwargs_with_deployment(
deployment=router.get_deployment_by_model_group_name(model_group_name="o3-deep-research"), kwargs=kwargs
)
tool_fields: Final = ("tools", "tool_choice", "parallel_tool_calls")
assert {name: kwargs[name] for name in tool_fields if name in kwargs} == expected_tool_fields
def test_credential_name_injected_as_tag():
"""
Test that litellm_credential_name from deployment litellm_params
@ -16004,6 +16095,7 @@ class _AnthropicMessagesFakeByteStream:
self._chunks = list(chunks)
self._hidden_params = {"additional_headers": {"x-amzn-requestid": "req-1"}}
self.closed = False
self.closed_as_superseded = False
def __aiter__(self):
return self
@ -16014,6 +16106,7 @@ class _AnthropicMessagesFakeByteStream:
return self._chunks.pop(0)
async def aclose(self) -> None:
self.closed_as_superseded = self.closed_as_superseded or closing_superseded_attempt()
self.closed = True
@ -17256,6 +17349,30 @@ async def test_anthropic_messages_stream_dropped_before_content_is_retried_withi
assert [(attempted, budget) for _, attempted, budget in provider.calls] == [(0, 2), (1, 2)]
@pytest.mark.asyncio
async def test_anthropic_messages_superseded_stream_is_closed_before_the_retry_opens():
"""Both attempts log through the request's one logging object, so the superseded attempt's end-of-stream
logging must be queued before the retry starts producing its own: the router closes the dropped stream
before it asks the group for another."""
router = _anthropic_messages_retry_router(num_retries=1)
superseded = _AnthropicMessagesFakeByteStream(
[_anthropic_messages_message_start_chunk(), _anthropic_messages_overloaded_error_chunk()]
)
superseded_closed_when_retry_opened: list[bool] = []
def retry_stream():
superseded_closed_when_retry_opened.append(superseded.closed_as_superseded)
return _anthropic_messages_retried_stream()
provider = _AnthropicMessagesScriptedProvider(lambda: superseded, retry_stream)
stream = await _anthropic_messages_stream_through_router(router, provider)
body = [chunk async for chunk in stream]
assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("pong")]
assert superseded_closed_when_retry_opened == [True]
@pytest.mark.asyncio
async def test_a_ptu_share_holders_messages_stream_dropped_before_content_is_retried_on_its_shared_group(monkeypatch):
"""The same-group retry has to look the group's health up as the calling team; looked up with no
@ -18387,7 +18504,7 @@ async def test_aanthropic_messages_recover_stream_error_flushes_buffered_frames_
with patch.object(router, "_aanthropic_messages_fallback_attempt") as mock_attempt:
recovery = router._aanthropic_messages_recover_stream_error(
declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper()
declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
)
with pytest.raises(BedrockError) as exc_info:
await drain(recovery)
@ -18408,7 +18525,7 @@ async def test_aanthropic_messages_recover_stream_error_hands_converted_raise_to
with patch.object(router, "_aanthropic_messages_fallback_attempt", new=fake_attempt):
recovery = router._aanthropic_messages_recover_stream_error(
raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper()
raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
)
collected = [chunk async for chunk in recovery]
assert collected == [b"fallback"]
@ -18466,6 +18583,25 @@ async def test_anthropic_messages_post_first_chunk_error_skips_fallback():
mock_fallback.assert_not_awaited()
@pytest.mark.asyncio
async def test_anthropic_messages_stream_the_client_closes_at_a_forwarded_error_is_not_closed_as_superseded():
"""An error frame forwarded to the client ends that request, and the SDK hangs up on it. No retry took the
request over, so the source must not be closed as a superseded attempt, which would log it as a failure and
drop the usage the provider billed for it."""
router = _anthropic_messages_make_router()
error_chunk = _anthropic_messages_overloaded_error_chunk()
source = _AnthropicMessagesFakeByteStream([_anthropic_messages_content_chunk("partial answer"), error_chunk])
wrapped = await router._aanthropic_messages_streaming_iterator(response=source, initial_kwargs={"model": "primary"})
async for chunk in wrapped:
if chunk == error_chunk:
break
await wrapped.aclose()
assert source.closed is True
assert source.closed_as_superseded is False
@pytest.mark.asyncio
async def test_anthropic_messages_non_retriable_error_flushes_buffered_lifecycle_frames():
"""A non-retriable error arriving while lifecycle frames are still

View file

@ -349,6 +349,56 @@ async def test_malformed_vector_store_index_row_from_the_db_raises_a_validation_
await VectorStoreIndexRegistry.get_vector_store_indexes_from_db(_database_listing_index_rows([row]))
@pytest.mark.parametrize(
("tools", "expected"),
[
(5, []),
(["vector_store_ids", {"type": "file_search", "vector_store_ids": ["vs_kept"]}], ["vs_kept"]),
(
[{"type": "file_search", "vector_store_ids": 5}, {"type": "file_search", "vector_store_ids": ["vs_kept"]}],
["vs_kept"],
),
],
ids=["int", "string_entry", "int_vector_store_ids"],
)
def test_vector_store_ids_to_run_skips_malformed_tools(tools: object, expected: list[str]) -> None:
assert VectorStoreRegistry().get_vector_store_ids_to_run(non_default_params={}, tools=tools) == expected
@pytest.mark.parametrize(
"vector_store_ids",
["vs_requested", ["vs_requested", 5]],
ids=["string", "list_with_a_non_string"],
)
def test_vector_store_ids_to_run_reads_every_store_id_a_malformed_vector_store_ids_field_names(
vector_store_ids: object,
) -> None:
tools: Final = [{"type": "file_search", "vector_store_ids": vector_store_ids}]
assert VectorStoreRegistry().get_vector_store_ids_to_run(non_default_params={}, tools=tools) == ["vs_requested"]
@pytest.mark.parametrize(
"vector_store_ids",
["vs_allowed", ["vs_allowed", 12345], [12345], ["vs_allowed"]],
ids=["string", "list_with_a_non_string", "only_non_strings", "list_of_strings"],
)
def test_vector_store_tools_pop_exactly_the_store_ids_the_registry_reports_to_run(vector_store_ids: object) -> None:
now: Final = datetime.now(timezone.utc)
registry: Final = VectorStoreRegistry(
[
LiteLLM_ManagedVectorStore(vector_store_id=vector_store_id, created_at=now, updated_at=now)
for vector_store_id in ("vs_allowed", 12345)
]
)
tools: Final = [{"type": "file_search", "vector_store_ids": vector_store_ids}]
checked: Final = registry.get_vector_store_ids_to_run(non_default_params={}, tools=tools)
ran: Final[list[str]] = []
registry.get_and_pop_recognised_vector_store_tools(tools=[dict(tool) for tool in tools], vector_store_ids=ran)
assert ran == checked
class TestRegistriesDoNotShareDefaultState:
def test_adding_to_one_registry_leaves_the_next_one_empty(self):
VectorStoreRegistry().add_vector_store_to_registry(

View file

@ -3,9 +3,11 @@ import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { Label } from "@/components/ui/label";
import { PasswordInput } from "@/components/shared/PasswordInput";
import { Input } from "@/components/ui/input";
import { MultiSelect } from "@/components/shared/MultiSelect";
import DcrBridgeToggle from "./DcrBridgeToggle";
import { MountedFormField } from "@/components/common_components/MountedFormField";
import { textControl } from "./mcpFieldRules";
import { tagsControl, textControl } from "./mcpFieldRules";
import { credentialAuthClass, isClientForwardedTokenMode } from "@/components/mcp_tools/types";
interface PassthroughOAuthFlow {
@ -69,6 +71,7 @@ export default function PassthroughAuthorizeSection({
const clientSecretPlaceholder = classUnchanged
? "Leave blank to keep the currently saved secret (if any)"
: "Leave blank for public clients / PKCE";
const fieldClassName = "rounded-lg border-border focus:border-info focus:ring-ring";
const clientIdExtra = classUnchanged
? "Set this to make everyone authorize through a specific app; required for upstreams without dynamic client registration (e.g. a pre-registered Slack app)."
: "Switching the auth type discards the previously saved app; enter a client ID here or leave blank to use dynamic client registration.";
@ -113,6 +116,39 @@ export default function PassthroughAuthorizeSection({
/>
)}
</MountedFormField>
<MountedFormField
label={<span className="text-sm font-medium text-foreground">Authorization URL (optional)</span>}
name="authorization_url"
help="Set this when the upstream publishes no OAuth metadata, for example an internal server behind Microsoft Entra. Leave blank to discover it from the upstream."
>
{(control) => (
<Input
{...textControl(control)}
placeholder="https://login.microsoftonline.com/<tenant-id>/oauth2/v2.0/authorize"
className={fieldClassName}
/>
)}
</MountedFormField>
<MountedFormField
label={<span className="text-sm font-medium text-foreground">Token URL (optional)</span>}
name="token_url"
help="Token endpoint of the same authorization server. Leave blank to discover it from the upstream."
>
{(control) => (
<Input
{...textControl(control)}
placeholder="https://login.microsoftonline.com/<tenant-id>/oauth2/v2.0/token"
className={fieldClassName}
/>
)}
</MountedFormField>
<MountedFormField
label={<span className="text-sm font-medium text-foreground">Scopes (optional)</span>}
name={["credentials", "scopes"]}
help="Scopes requested from the upstream when a user authorizes. Press enter or comma between scopes."
>
{(control) => <MultiSelect {...tagsControl(control)} placeholder="Add scopes" className="rounded-lg" />}
</MountedFormField>
<DcrBridgeToggle authType={authType} initialChecked={dcrBridgeInitialChecked} />
{isEditing && onRemoveStoredAppChange && (
<Label className="items-start leading-normal font-normal text-foreground">

View file

@ -201,6 +201,22 @@ describe("buildCreateServerPayload credentials", () => {
},
);
it.each([["true_passthrough"], ["oauth_delegate"]])(
"carries the admin-entered authorization URL, token URL and scopes for %s",
(authType) => {
const entraServer = {
auth_type: authType,
authorization_url: "https://login.microsoftonline.com/tenant/oauth2/v2.0/authorize",
token_url: "https://login.microsoftonline.com/tenant/oauth2/v2.0/token",
credentials: { client_id: "org-app", scopes: ["api://mcp/.default", "offline_access", ""] },
};
const payload = payloadOf(build(entraServer));
expect(payload.authorization_url).toBe("https://login.microsoftonline.com/tenant/oauth2/v2.0/authorize");
expect(payload.token_url).toBe("https://login.microsoftonline.com/tenant/oauth2/v2.0/token");
expect(payload.credentials).toEqual({ client_id: "org-app", scopes: ["api://mcp/.default", "offline_access"] });
},
);
it("merges the DCR-minted client into an oauth2 payload", () => {
const payload = payloadOf(
build(

View file

@ -295,11 +295,6 @@ export const CASES: readonly DifferentialCase[] = [
ui: {},
},
{ label: "credentials absent entirely", values: { ...ROOT, auth_type: "oauth2" }, ui: {} },
{
label: "removeStoredApp forces an explicit-null app write",
values: { ...ROOT, auth_type: "true_passthrough", credentials: { ...CREDS } },
ui: { removeStoredApp: true, mcpServer: { ...SERVER, auth_type: "true_passthrough" } },
},
{
label: "removeStoredApp ignored outside a client-forwarded mode",
values: { ...ROOT, auth_type: "api_key", credentials: { ...CREDS } },

View file

@ -398,3 +398,61 @@ it.each([
expect(result.payload.credentials?.client_secret).toBe(useDcr ? null : "configured-secret");
expect(result.payload.credentials?.dcr_issuer).toBe(useDcr ? "https://new.example" : undefined);
});
it.each([AUTH_TYPE.OAUTH_DELEGATE, AUTH_TYPE.TRUE_PASSTHROUGH])(
"keeps the endpoints typed for %s when the server was saved as oauth2",
(authType) => {
const authorizationUrl = "https://login.microsoftonline.com/tenant/oauth2/v2.0/authorize";
const tokenUrl = "https://login.microsoftonline.com/tenant/oauth2/v2.0/token";
const values: EditServerFormValues = {
auth_type: authType,
transport: "http",
authorization_url: authorizationUrl,
token_url: tokenUrl,
};
const ui: EditServerUiState = { ...baseUi, mcpServer: { ...baseUi.mcpServer, auth_type: AUTH_TYPE.OAUTH2 } };
const result = buildEditServerPayload(values, ui);
expect(result.kind).toBe("ok");
if (result.kind !== "ok") throw new Error("Expected valid MCP server payload");
expect(result.payload.authorization_url).toBe(authorizationUrl);
expect(result.payload.token_url).toBe(tokenUrl);
expect(result.payload.issuer).toBeNull();
expect(result.payload.registration_url).toBeNull();
},
);
it.each([AUTH_TYPE.OAUTH_DELEGATE, AUTH_TYPE.TRUE_PASSTHROUGH])(
"writes an explicit null when the admin removes every scope on a %s server",
(authType) => {
const values: EditServerFormValues = {
auth_type: authType,
transport: "http",
credentials: { client_id: "org-app", scopes: [""] },
};
const ui: EditServerUiState = { ...baseUi, mcpServer: { ...baseUi.mcpServer, auth_type: authType } };
const result = buildEditServerPayload(values, ui);
expect(result.kind).toBe("ok");
if (result.kind !== "ok") throw new Error("Expected valid MCP server payload");
expect(result.payload.credentials).toEqual({ client_id: "org-app", scopes: null });
},
);
it.each([
[["read"], ["read"]],
[[""], null],
])("keeps the scope edit %j when the saved app is removed on a client-forwarded server", (typed, expected) => {
const values: EditServerFormValues = {
auth_type: AUTH_TYPE.TRUE_PASSTHROUGH,
transport: "http",
credentials: { client_id: "cid", client_secret: "csec", scopes: typed },
};
const ui: EditServerUiState = {
...baseUi,
removeStoredApp: true,
mcpServer: { ...baseUi.mcpServer, auth_type: AUTH_TYPE.TRUE_PASSTHROUGH },
};
const result = buildEditServerPayload(values, ui);
expect(result.kind).toBe("ok");
if (result.kind !== "ok") throw new Error("Expected valid MCP server payload");
expect(result.payload.credentials).toEqual({ scopes: expected, client_id: null, client_secret: null });
});

View file

@ -214,6 +214,21 @@ const buildCredentials = (credentialValues: unknown): Readonly<Record<string, un
return Object.fromEntries(kept);
};
const unmountedOauth2Endpoints = (authType: string | undefined) =>
isClientForwardedTokenMode(authType) ? {} : { authorization_url: null, token_url: null };
const scopesCleared = (credentialValues: unknown): boolean => {
if (!credentialValues || typeof credentialValues !== "object") return false;
const scopes: unknown = (credentialValues as { scopes?: unknown }).scopes;
return Array.isArray(scopes) && scopes.every((scope: unknown) => scope == null || scope === "");
};
const withClearedScopes = (
credentials: Readonly<Record<string, unknown>> | undefined,
credentialValues: unknown,
): Readonly<Record<string, unknown>> | undefined =>
scopesCleared(credentialValues) ? { ...(credentials ?? {}), scopes: null } : credentials;
interface CredentialsEntryInput {
readonly dcrClient?: McpDcrCredentials | null;
readonly oauthFlow?: string;
@ -239,7 +254,7 @@ const resolveCredentialsEntry = ({
return { credentials: { ...credentials, ...dcrClient } };
}
if (removeStoredApp && isClientForwardedTokenMode(authType)) {
return { credentials: { client_id: null, client_secret: null } };
return { credentials: { ...(credentials ?? {}), client_id: null, client_secret: null } };
}
if (includeCredentials && credentials && Object.keys(credentials).length > 0) {
return { credentials };
@ -334,7 +349,7 @@ export const buildEditServerPayload = (values: EditServerFormValues, ui: EditSer
// Client-forwarded rows persist ONLY the declared app; strip any token material lingering in the
// form (e.g. from a prior oauth2 authorize this session) so it can never reach the row.
const submitCredentials = isClientForwardedTokenMode(restValues.auth_type)
? preservedAdminCredentials(credentialsPayload)
? withClearedScopes(preservedAdminCredentials(credentialsPayload), credentialValues)
: credentialsPayload;
const includeCredentials = restValues.auth_type && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(restValues.auth_type);
@ -355,7 +370,7 @@ export const buildEditServerPayload = (values: EditServerFormValues, ui: EditSer
stdio_config: undefined,
env_json: undefined,
...(mcpServer.auth_type === AUTH_TYPE.OAUTH2 && restValues.auth_type !== AUTH_TYPE.OAUTH2
? { issuer: null, authorization_url: null, token_url: null, registration_url: null }
? { issuer: null, ...unmountedOauth2Endpoints(restValues.auth_type), registration_url: null }
: {}),
...(mcpServer.auth_type === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE &&
restValues.auth_type !== AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE

View file

@ -264,6 +264,8 @@ const CASES: readonly Case[] = [
...EXPECTED_BASE,
auth_type: "true_passthrough",
credentials: { client_id: "cid", client_secret: "csec" },
authorization_url: undefined,
token_url: undefined,
},
},
{
@ -273,6 +275,8 @@ const CASES: readonly Case[] = [
...EXPECTED_BASE,
auth_type: "oauth_delegate",
credentials: { client_id: "cid", client_secret: "csec" },
authorization_url: undefined,
token_url: undefined,
},
},
{
@ -283,7 +287,13 @@ const CASES: readonly Case[] = [
{
label: "dcr_bridge stays true for a client-forwarded mode",
server: { ...BASE, auth_type: "true_passthrough", dcr_bridge: true },
expected: { ...EXPECTED_BASE, auth_type: "true_passthrough", dcr_bridge: true },
expected: {
...EXPECTED_BASE,
auth_type: "true_passthrough",
dcr_bridge: true,
authorization_url: undefined,
token_url: undefined,
},
},
{
label: "an existing allowed_tools list enforces the allowlist and round-trips",

View file

@ -333,6 +333,14 @@ describe("preservedAdminCredentials vs preservedDeclaredAppCredentials", () => {
expect(preservedAdminCredentials(credentials)).toEqual({ client_id: "cid", upstream_resource: "auto" });
});
it("preserves admin-typed scopes and drops blank ones", () => {
expect(preservedAdminCredentials({ client_id: "cid", scopes: ["api://mcp/.default", ""] })).toEqual({
client_id: "cid",
scopes: ["api://mcp/.default"],
});
expect(preservedAdminCredentials({ client_id: "cid", scopes: [] })).toEqual({ client_id: "cid" });
});
// The two helpers answer different questions and must not be collapsed: "has the admin declared an
// OAuth app" gates the app-may-not-match-upstream warning, so a resource-only server must read as
// having no declared app.

View file

@ -181,10 +181,23 @@ export const preservedDeclaredAppCredentials = (
// Everything the admin typed into `credentials` and nothing minted: the declared app plus the config
// keys. This is what must survive the invalidation reset and what a client-forwarded row may persist,
// so dropping a key from here silently discards admin input on an unrelated edit.
const adminScopes = (value: unknown): string[] | undefined => {
if (!Array.isArray(value)) return undefined;
const kept = value.filter((scope): scope is string => typeof scope === "string" && scope !== "");
return kept.length > 0 ? kept : undefined;
};
export const preservedAdminCredentials = (
credentials: Record<string, unknown> | null | undefined,
): Record<string, string> | undefined =>
pickStringCredentials(credentials, [...DECLARED_APP_CREDENTIAL_KEYS, ...ADMIN_CONFIG_CREDENTIAL_KEYS]);
): Record<string, string | string[]> | undefined => {
const strings = pickStringCredentials(credentials, [
...DECLARED_APP_CREDENTIAL_KEYS,
...ADMIN_CONFIG_CREDENTIAL_KEYS,
]);
const scopes = adminScopes(credentials?.scopes);
if (scopes === undefined) return strings;
return { ...(strings ?? {}), scopes };
};
// Drop minted token keys, keeping everything else (the declared app plus any non-token config).
export const withoutMintedTokenCredentials = (

View file

@ -26991,6 +26991,12 @@ export interface components {
* @default true
*/
fail_on_error: boolean | null;
/**
* Forward Messages As Sent
* @description Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can carry provider-native content blocks and keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with undeclared keys removed.
* @default false
*/
forward_messages_as_sent: boolean | null;
/**
* Guard Name
* @description Name of the guardrail in guardrails.ai
@ -36471,6 +36477,12 @@ export interface components {
* @description HTTP timeout in seconds for checking attached files. Default: 10.
*/
file_guardrail_timeout?: number | null;
/**
* Forward Messages As Sent
* @description Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can carry provider-native content blocks and keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with undeclared keys removed.
* @default false
*/
forward_messages_as_sent: boolean | null;
/**
* Gateway Name
* @description noma_v2 only: name of this gateway, used as the gateway_host label on Noma scans