diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index fbdbfafa66d..b442948c3ca 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -32,6 +32,7 @@ IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)") DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*") COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE) GLOB_CHARS = frozenset("*?") +DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"}) # Trees whose jobs are sharded with no catch-all bucket, so every child that holds # tests has to be named by some shard or it runs nowhere. A child listed here is @@ -560,6 +561,17 @@ def _integration_github_files(runner: pathlib.Path) -> frozenset[str]: return frozenset(ast.literal_eval(values)) +def _owns(workflow: str, relative_path: str) -> bool: + owned_root: Final = DEDICATED_INTEGRATION_WORKFLOWS.get(workflow) + return owned_root is not None and relative_path.startswith(owned_root) + + +def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]: + return frozenset[str]().union( + *(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path)) + ) + + def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]: runner: Final = repo_root / "tests/integration/run.py" if not runner.exists(): @@ -611,15 +623,20 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens ) != 1 ) - gha_tokens: Final = _invoked_test_tokens( - scalar - for path in (repo_root / ".github/workflows").glob("*.y*ml") - for scalar in _scalars(yaml.safe_load(path.read_text()), path.name) + workflow_tokens: Final = MappingProxyType( + { + path.name: _invoked_test_tokens(_scalars(yaml.safe_load(path.read_text()), path.name)) + for path in (repo_root / ".github/workflows").glob("*.y*ml") + } + ) + gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values()) + circle_owned: Final = ( + frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files ) findings: Final = ( tuple( - Finding(path, "integration contract is also selected by GitHub Actions") - for path in paths + Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions") + for path in sorted(circle_owned) if any(_token_covers(token, path) for token in gha_tokens) ) + tuple( @@ -632,6 +649,11 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens for path in sorted(github_files) if not (repo_root / path).is_file() ) + + tuple( + Finding(path, "GitHub-owned integration contract runs outside a dedicated integration workflow") + for path in sorted(github_files) + if any(_token_covers(token, path) for token in _tokens_outside_owner(path, workflow_tokens)) + ) ) browser_commands: Final = tuple( scalar.value @@ -688,13 +710,9 @@ def main() -> int: scalars = _all_scalars() integration_paths, ownership_findings = _integration_ownership() - test_findings = ( - _uncovered_tests( - allowlist, - _invoked_selections(scalars) - + (Selection(included=integration_paths, ignored=frozenset()),), - ) - + ownership_findings + test_findings = _uncovered_tests( + allowlist, + _invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),), ) dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars)) stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles()) @@ -711,13 +729,25 @@ def main() -> int: test_findings, "Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.", ) + if ownership_findings: + browser_remedy = ( + " CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may " + "select them and the default Playwright config must exclude them." + if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings) + else "" + ) + _report( + "integration contracts with the wrong CI owner", + ownership_findings, + "Follow tests/integration/AGENTS.md, which says which CI job owns each integration file." + browser_remedy, + ) if dockerfile_findings: _report( "Dockerfiles that no CI job builds", dockerfile_findings, "Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.", ) - if stale_findings or test_findings or dockerfile_findings: + if stale_findings or test_findings or ownership_findings or dockerfile_findings: return 1 _write( diff --git a/AGENTS.md b/AGENTS.md index 87e4df49299..f4b1de57807 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,7 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a `tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_.py` if you're the first test there). One focused regression test beats many shallow ones -End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md` +End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD` @@ -72,7 +72,7 @@ When referencing or running models (coding, QA'ing, writing docs, writing tests, Always pull before starting any work. The checkout or worktree may be sitting on a stale branch -If you're an internal contributor, when creating a new PR, the typical flow is to branch off the repository's current default branch and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names +Internal contributors branch a new PR off the repository's current default branch, with a branch name prefixed with litellm_ Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch diff --git a/litellm/__init__.py b/litellm/__init__.py index 88e56c14187..83e7642c39f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1416,6 +1416,7 @@ from .exceptions import ( InvalidRequestError, BadRequestError, ImageFetchError, + MalformedRequestFieldError as MalformedRequestFieldError, VectorStoreSearchError, NotFoundError, PermissionDeniedError, diff --git a/litellm/constants.py b/litellm/constants.py index 39363cc02e4..425fc22d340 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1859,6 +1859,7 @@ SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_L PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25)) PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0)) PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0)) +TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS: Final = 30 DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = get_env_int("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60) # 1 minute PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597) RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, get_env_int("RESET_BUDGET_JOB_BATCH_SIZE", 500)) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index b3a0802db13..733f1aa72d5 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -313,6 +313,10 @@ class VectorStoreSearchError(BadRequestError): ) +class MalformedRequestFieldError(BadRequestError): + pass + + class UnprocessableEntityError(openai.UnprocessableEntityError): def __init__( self, diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index fa5496004e3..97a1073d7c0 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -27,6 +27,7 @@ from litellm.constants import ( PROXY_REJECTED_BEFORE_ROUTING_KEY, ) from litellm.exceptions import ( + MalformedRequestFieldError, validate_rate_limit_category, validate_rate_limit_type, ) @@ -3104,7 +3105,11 @@ class PrometheusLogger(CustomLogger): # On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"] # into requested_model and leave deployment-scoped labels empty. - deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY) + deployment_selected: Final = ( + bool(model_id) + and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY) + and not isinstance(exception, MalformedRequestFieldError) + ) if deployment_selected: label_litellm_model_name = litellm_model_name label_model_id = model_id diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 67d1573dbc4..241108b4c2f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2300,6 +2300,13 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["combined_usage_object"] = usage self.model_call_details["response_cost"] = response_cost + def attempt_scoped_copy(self) -> "Logging": + """A copy with its own model_call_details, so logging one attempt of a request through it leaves + the state the request's other attempts share untouched.""" + attempt: Final = copy.copy(self) + attempt.model_call_details = dict(self.model_call_details) + return attempt + def carry_failed_attempt_usage(self) -> None: """Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its own usage while the request's failure log still bills every attempt.""" diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index ce5dceb2e5c..04510c20db0 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -858,9 +858,9 @@ class StandardBuiltInToolCostTracking: @staticmethod def get_web_search_options(kwargs: Mapping[str, object]) -> WebSearchOptions | None: - if "web_search_options" in kwargs: - web_search_options: Final = cast(WebSearchOptions, kwargs.get("web_search_options", {})) - return WebSearchOptions(**web_search_options) + web_search_options: Final = kwargs.get("web_search_options") + if isinstance(web_search_options, dict): + return WebSearchOptions(**cast(WebSearchOptions, web_search_options)) tools: Final = StandardBuiltInToolCostTracking._get_tools_from_kwargs( kwargs=kwargs, tool_type="web_search_preview" @@ -876,10 +876,9 @@ class StandardBuiltInToolCostTracking: _get_web_search_options = get_web_search_options @staticmethod - def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> list[object] | None: - if "tools" in kwargs: - return cast(list[object], kwargs.get("tools", [])) - return None + def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> Sequence[object] | None: + tools: Final = kwargs.get("tools") + return cast(Sequence[object], tools) if isinstance(tools, (list, tuple)) else None @staticmethod def get_file_search_tool_call(kwargs: Mapping[str, object]) -> FileSearchTool | None: diff --git a/litellm/litellm_core_utils/tool_fields.py b/litellm/litellm_core_utils/tool_fields.py new file mode 100644 index 00000000000..f90c8f5d174 --- /dev/null +++ b/litellm/litellm_core_utils/tool_fields.py @@ -0,0 +1,164 @@ +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from functools import reduce +from typing import Annotated, Final, TypeAlias + +from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError +from typing_extensions import ReadOnly, TypedDict + +import litellm +from litellm.litellm_core_utils.core_helpers import normalize_drop_params + +ROUTES_WITH_TOOLS: Final = frozenset( + { + "acompletion", + "aresponses", + "acompact_responses", + "anthropic_messages", + "agenerate_content", + "agenerate_content_stream", + "acreate_interaction", + } +) +TOOL_FIELDS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls", "toolConfig", "tool_config"}) +_TOOLS: Final = TypeAdapter(list[object]) +_FIELD_LIST: Final = TypeAdapter(list[object], config=ConfigDict(strict=True)) +_JSON_OBJECT: Final = TypeAdapter(Json[dict[str, object]]) + + +def _schema_as_read(value: object) -> object: + if not value: + return None + if not isinstance(value, str): + return value + try: + return _JSON_OBJECT.validate_python(value) + except ValidationError: + return value + + +_Schema: TypeAlias = Annotated[dict[str, object] | None, BeforeValidator(_schema_as_read)] + + +class _ToolFunction(TypedDict, total=False): + parameters: ReadOnly[_Schema] + + +class _ToolEntry(TypedDict, total=False): + function: ReadOnly[_ToolFunction | None] + parameters: ReadOnly[_Schema] + input_schema: ReadOnly[_Schema] + vector_store_ids: ReadOnly[list[str] | None] + + +_TOOL_ENTRY: Final = TypeAdapter(_ToolEntry) +_WEB_SEARCH_OPTIONS: Final = TypeAdapter(dict[str, object] | None, config=ConfigDict(strict=True)) + + +@dataclass(frozen=True, slots=True) +class DropPolicy: + drop_params: bool + additional_drop_params: frozenset[str] + + def drops(self, field: str) -> bool: + return self.drop_params or field in self.additional_drop_params + + +@dataclass(frozen=True, slots=True) +class MalformedField: + field: str + message: str + dropped: Callable[[Mapping[str, object]], Mapping[str, object]] + + +def _field_names(value: object) -> frozenset[str]: + try: + return frozenset(name for name in _FIELD_LIST.validate_python(value) if isinstance(name, str)) + except ValidationError: + return frozenset() + + +def drop_policy(params: Mapping[str, object]) -> DropPolicy: + return DropPolicy( + drop_params=litellm.drop_params is True or normalize_drop_params(params.get("drop_params")) is True, + additional_drop_params=_field_names(params.get("additional_drop_params")), + ) + + +def first_validation_error(e: ValidationError, prefix: tuple[str | int, ...] = ()) -> str: + error: Final = e.errors(include_url=False)[0] + return f"{'.'.join(str(part) for part in (*prefix, *error['loc']))}: {error['msg']}" + + +def _tool_entry_error(index: int, entry: object) -> str | None: + try: + _TOOL_ENTRY.validate_python(entry, strict=True) + except ValidationError as e: + return first_validation_error(e, prefix=("tools", index)) + return None + + +def _without_tool_fields(data: Mapping[str, object]) -> Mapping[str, object]: + return {name: value for name, value in data.items() if name not in TOOL_FIELDS} + + +def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None: + tools: Final = data.get("tools") + if tools is None: + return None + try: + entries: Final = _TOOLS.validate_python(tools) + except ValidationError as e: + return MalformedField("tools", first_validation_error(e, prefix=("tools",)), _without_tool_fields) + entry_errors: Final = tuple(_tool_entry_error(index, entry) for index, entry in enumerate(entries)) + first_error: Final = next((error for error in entry_errors if error is not None), None) + if first_error is None: + return None + kept: Final = [entry for entry, error in zip(entries, entry_errors) if error is None] + return MalformedField( + "tools", first_error, lambda fields: {**fields, "tools": kept} if kept else _without_tool_fields(fields) + ) + + +def _without_web_search_options(data: Mapping[str, object]) -> Mapping[str, object]: + return {name: value for name, value in data.items() if name != "web_search_options"} + + +def _malformed_web_search_options(data: Mapping[str, object]) -> MalformedField | None: + try: + _WEB_SEARCH_OPTIONS.validate_python(data.get("web_search_options")) + except ValidationError as e: + return MalformedField( + "web_search_options", + first_validation_error(e, prefix=("web_search_options",)), + _without_web_search_options, + ) + return None + + +def resolve_tool_fields( + data: Mapping[str, object], candidates: Callable[[], tuple[DropPolicy, ...]] +) -> Mapping[str, object] | MalformedField: + checked: Final = (_malformed_tools(data), _malformed_web_search_options(data)) + malformed: Final = tuple(field for field in checked if field is not None) + if not malformed: + return data + policies: Final = candidates() + rejected: Final = next( + (field for field in malformed if not any(policy.drops(field.field) for policy in policies)), None + ) + if rejected is not None: + return rejected + return reduce( + lambda resolved, field: field.dropped(resolved), + (field for field in malformed if all(policy.drops(field.field) for policy in policies)), + data, + ) + + +def resolve_deployment_tool_fields( + kwargs: Mapping[str, object], route: object +) -> Mapping[str, object] | MalformedField: + if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS: + return kwargs + return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),)) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 6bc1c4ead64..6b6974becac 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -45,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, effective_skip_tool_message_for_guardrail, + list_field, merge_guardrailed_scoped_messages, merge_returned_tools_into_request_tools, scoped_structured_message_indices, @@ -924,10 +925,10 @@ class AnthropicMessagesHandler(BaseTranslation): images=(), ) - def extract_request_tool_names(self, data: dict) -> list[str]: + def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]: """Extract every tool name in an Anthropic messages request: tools[].name, plus tools[].function.name for OpenAI-format tools the bridge forwards verbatim.""" - return [name for tool in data.get("tools") or [] for name in anthropic_tool_names(tool)] + return [name for tool in list_field(data, "tools") for name in anthropic_tool_names(tool)] @classmethod def _extract_input_text_and_images( diff --git a/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py index 5fb2546a5ae..e65fe596da7 100644 --- a/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py @@ -2,6 +2,7 @@ import asyncio import json import re from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence +from contextvars import ContextVar from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable @@ -352,6 +353,22 @@ async def aclose_if_supported(stream: object) -> None: await stream.aclose() +_CLOSING_SUPERSEDED_ATTEMPT: Final[ContextVar[bool]] = ContextVar("closing_superseded_attempt", default=False) + + +async def aclose_superseded_attempt(stream: object) -> None: + """Close an attempt a retry or fallback is taking over, so its end-of-stream logging records a failure.""" + token: Final = _CLOSING_SUPERSEDED_ATTEMPT.set(True) + try: + await aclose_if_supported(stream) + finally: + _CLOSING_SUPERSEDED_ATTEMPT.reset(token) + + +def closing_superseded_attempt() -> bool: + return _CLOSING_SUPERSEDED_ATTEMPT.get() + + _RESPONSE_HEADERS_ADAPTER: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str]) diff --git a/litellm/llms/base_llm/guardrail_translation/utils.py b/litellm/llms/base_llm/guardrail_translation/utils.py index 39ff2fa0528..55a4b3da0b3 100644 --- a/litellm/llms/base_llm/guardrail_translation/utils.py +++ b/litellm/llms/base_llm/guardrail_translation/utils.py @@ -4,7 +4,7 @@ import json from collections.abc import Callable, Iterator, Mapping, Sequence from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage @@ -292,6 +292,17 @@ def openai_tool_name(tool: object) -> str | None: return flat_name if isinstance(flat_name, str) else None +_FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True)) +_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True)) + + +def list_field(data: object, key: str) -> Sequence[object]: + try: + return _ITEMS.validate_python(_FIELDS.validate_python(data).get(key)) + except ValidationError: + return () + + def anthropic_tool_names(tool: object) -> tuple[str, ...]: """Every name a /v1/messages tool dict can act under: the flat Anthropic ``name`` plus ``function.name`` for OpenAI-format tools the bridge forwards verbatim. Allowlist checks diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index c168c3f88fc..3070bedd30d 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -39,6 +39,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( effective_scan_only_tool_results_for_guardrail, effective_skip_system_message_for_guardrail, effective_skip_tool_message_for_guardrail, + list_field, merge_guardrailed_scoped_messages, merge_returned_tools_into_request_tools, openai_tool_name, @@ -156,7 +157,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs["structured_messages"] = [structured_messages[index] for index in scoped_message_indices] # Pass tools (function definitions) to the guardrail tools: Final = data.get("tools") - if tools and not scan_only_tool_results: + if isinstance(tools, list) and tools and not scan_only_tool_results: inputs["tools"] = tools # Include model information if available model: Final = data.get("model") @@ -258,19 +259,19 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return "no scannable content after message scoping" return None if images else "no scannable content" - def extract_request_tool_names(self, data: dict) -> list[str]: + def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]: """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name, and the tools a replayed tool_search_output loads).""" names: Final[list[str]] = [] - for tool in data.get("tools") or []: + for tool in list_field(data, "tools"): if isinstance(tool, dict) and tool.get("type") == "function": fn = tool.get("function") if isinstance(fn, dict) and fn.get("name"): names.append(str(fn["name"])) - for fn in data.get("functions") or []: + for fn in list_field(data, "functions"): if isinstance(fn, dict) and fn.get("name"): names.append(str(fn["name"])) - return [*names, *replayed_tool_search_tool_names(cast(object, data))] # cast-ok: upcast, read field by field + return [*names, *replayed_tool_search_tool_names(data)] def _extract_inputs( self, diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 435c11c0602..84ff790ae48 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -55,6 +55,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import ( from litellm.llms.base_llm.guardrail_translation.utils import ( blocked_responses_stream_usage, effective_skip_system_message_for_guardrail, + list_field, merge_guardrailed_scoped_messages, role_out_of_guardrail_scope, scoped_structured_message_indices, @@ -764,11 +765,11 @@ class OpenAIResponsesHandler(BaseTranslation): data.get("input"), data.get("instructions"), structured_messages, merged ) - def extract_request_tool_names(self, data: dict) -> list[str]: + def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]: """Extract tool names from Responses API request (tools[].name for function and custom, tools[].server_label for mcp).""" names: Final[list[str]] = [] - for tool in data.get("tools") or []: + for tool in list_field(data, "tools"): if not isinstance(tool, dict): continue if tool.get("type") in ("function", "custom") and tool.get("name"): diff --git a/litellm/proxy/_experimental/mcp_server/subscriptions.py b/litellm/proxy/_experimental/mcp_server/subscriptions.py index dcf24f974fe..db622ea3cbf 100644 --- a/litellm/proxy/_experimental/mcp_server/subscriptions.py +++ b/litellm/proxy/_experimental/mcp_server/subscriptions.py @@ -55,6 +55,7 @@ from litellm.proxy._experimental.mcp_server.catalog import ( ) from litellm.proxy._experimental.mcp_server.contracts import OperationContext from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError from litellm.types.mcp import MCP_LEGACY_VERSIONS from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -300,6 +301,29 @@ async def _listing(event: ServerEvent, context: OperationContext) -> str | None: raise MCPError(code=-32603, message="Subscription catalog exceeds the pagination limit") +async def _enforce_rate_limit(context: OperationContext, server: MCPServer) -> None: + await operations._enforce_mcp_server_rate_limit(context.user_api_key_auth, server) # pyright: ignore[reportPrivateUsage] # same admission policy as other gateway operations + + +async def _unmetered(_: OperationContext, __: MCPServer) -> None: + return None + + +async def _admitted() -> ProxyRateLimitError | None: + return None + + +@asynccontextmanager +async def _listener_admission(targets: tuple[SubscriptionTarget, ...]) -> AsyncGenerator[None]: + scope: Final = operations._mcp_server_admission_memo.set( # pyright: ignore[reportPrivateUsage] # the listener's own catalog refreshes reuse its admission + {target.server.server_id: asyncio.create_task(_admitted()) for target in targets} + ) + try: + yield + finally: + operations._mcp_server_admission_memo.reset(scope) # pyright: ignore[reportPrivateUsage] # restore the caller's admission scope + + async def _recheck_authorization(check: Callable[[], Awaitable[OperationContext]]) -> None: while True: await asyncio.sleep(15) @@ -361,7 +385,7 @@ class GatewaySubscriptions: if ctx.protocol_version != "2026-07-28": raise MCPError(code=-32601, message="Subscriptions require the modern MCP protocol") async with self._capacity(owner): - return await self._listen(ctx, params, authorize) + return await self._listen(ctx, params, authorize, _enforce_rate_limit) @asynccontextmanager async def _capacity(self, owner: str) -> AsyncGenerator[None]: @@ -414,6 +438,7 @@ class GatewaySubscriptions: ctx, SubscriptionFilter(tools_list_changed=True, prompts_list_changed=True, resources_list_changed=True), authorization, + _unmetered, ) if (ready := self._initializing.get(key)) is not None: ready.set() @@ -427,6 +452,7 @@ class GatewaySubscriptions: ctx: ServerRequestContext, notifications: SubscriptionFilter, authorization: SubscriptionAuthorization, + admit: Callable[[OperationContext, MCPServer], Awaitable[None]], scope: anyio.CancelScope | None = None, *, task_status: TaskStatus[None], @@ -443,7 +469,7 @@ class GatewaySubscriptions: async with self._capacity(authorization.owner): await self._listen( - ctx, SubscriptionsListenRequestParams(notifications=notifications), authorization, ready + ctx, SubscriptionsListenRequestParams(notifications=notifications), authorization, admit, ready ) async def legacy_resource(self, ctx: ServerRequestContext, uri: str, *, subscribe: bool) -> None: @@ -464,6 +490,7 @@ class GatewaySubscriptions: ctx, SubscriptionFilter(resource_subscriptions=[uri]), state.authorization, + _enforce_rate_limit, scope, ) state.resources = {**state.resources, uri: scope} @@ -476,6 +503,7 @@ class GatewaySubscriptions: ctx: ServerRequestContext, params: SubscriptionsListenRequestParams, authorize: Callable[[], Awaitable[OperationContext]], + admit: Callable[[OperationContext, MCPServer], Awaitable[None]], ready: Callable[[SubscriptionFilter], None] | None = None, ) -> SubscriptionsListenResult: context: Final = await authorize() @@ -501,7 +529,7 @@ class GatewaySubscriptions: return fresh async def relay(target: SubscriptionTarget, *, task_status: TaskStatus[SubscriptionFilter]) -> None: - await operations._enforce_mcp_server_rate_limit(context.user_api_key_auth, target.server) # pyright: ignore[reportPrivateUsage] # same admission policy as other gateway operations + await admit(context, target.server) legacy: Final = LegacyNotifications() @@ -567,7 +595,7 @@ class GatewaySubscriptions: finally: self._upstream_slots.release() - async with anyio.create_task_group() as group: + async with _listener_admission(targets), anyio.create_task_group() as group: honored: Final = tuple( [SubscriptionFilter.model_validate(await group.start(relay, target)) for target in targets] ) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 467cb2d624c..961d32ab664 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -14261,6 +14261,19 @@ "description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.", "title": "Fail On Error" }, + "forward_messages_as_sent": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can carry provider-native content blocks and keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with undeclared keys removed.", + "title": "Forward Messages As Sent" + }, "guard_name": { "anyOf": [ { @@ -16612,6 +16625,19 @@ "description": "HTTP timeout in seconds for checking attached files. Default: 10.", "title": "File Guardrail Timeout" }, + "forward_messages_as_sent": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can carry provider-native content blocks and keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with undeclared keys removed.", + "title": "Forward Messages As Sent" + }, "gateway_name": { "anyOf": [ { diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index a11268ad432..1fa276d9ab2 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -6,9 +6,9 @@ from collections.abc import Mapping from types import MappingProxyType from typing import Final -from fastapi import APIRouter, Depends, HTTPException, Request, Response +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import JSONResponse -from pydantic import JsonValue +from pydantic import JsonValue, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -18,6 +18,7 @@ from litellm.anthropic_interface.exceptions import ( AnthropicExceptionMapping, ) from litellm.integrations.custom_guardrail import ModifyResponseException +from litellm.litellm_core_utils.tool_fields import first_validation_error from litellm.llms.anthropic.pass_through.context_management import ( AnthropicContextManagementError, ) @@ -32,6 +33,7 @@ from litellm.proxy.common_request_processing import ( log_llm_api_exception, proxy_exception_from_http_exception, proxy_exception_from_route_error, + resolve_counted_tool_fields, resolve_litellm_call_id, ) from litellm.proxy.common_utils.error_body_call_id import error_body_call_id @@ -47,6 +49,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( with_litellm_call_id, ) from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli +from litellm.router import Router router: Final = APIRouter() @@ -91,6 +94,31 @@ def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSO return JSONResponse(status_code=status_code, content=content, headers=exc.headers) +_TOKEN_COUNT_FIELDS: Final = ("model", "messages", "tools", "system") + + +def _token_count_request(fields: Mapping[str, object]) -> TokenCountRequest | ProxyException: + try: + return TokenCountRequest.model_validate({name: fields[name] for name in _TOKEN_COUNT_FIELDS if name in fields}) + except ValidationError as e: + return ProxyException( + message=first_validation_error(e), + type="invalid_request_error", + param=str(e.errors(include_url=False)[0]["loc"][0]), + code=status.HTTP_400_BAD_REQUEST, + ) + + +def _token_count_fields( + data: Mapping[str, object], auto_drop_params: bool, llm_router: Router | None, team_id: str | None +) -> Mapping[str, object] | ProxyException: + fields: Final = { + name: data[name] for name in (*_TOKEN_COUNT_FIELDS, "drop_params", "additional_drop_params") if name in data + } + resolving: Final = {**fields, "drop_params": True} if auto_drop_params else fields + return resolve_counted_tool_fields(resolving, llm_router, team_id) + + def _strip_total_tokens_from_anthropic_response(response: object) -> None: """Remove the OpenAI-flavored `usage.total_tokens` field that LiteLLM injects into Anthropic /v1/messages responses. @@ -351,7 +379,7 @@ async def count_tokens( Returns: {"input_tokens": } """ - from litellm.proxy.proxy_server import count_request_tokens, proxy_config + from litellm.proxy.proxy_server import count_request_tokens, llm_router, proxy_config litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id")) try: @@ -368,26 +396,21 @@ async def count_tokens( if not messages: raise HTTPException(status_code=400, detail="messages parameter is required") - # Create TokenCountRequest for the internal endpoint - from litellm.proxy._types import TokenCountRequest - - token_request: Final = TokenCountRequest( - model=model_name, - messages=messages, - tools=data.get("tools"), - system=data.get("system"), + auto_drop_params: Final = should_auto_drop_params_for_agentic_cli( + request.headers.get("user-agent", ""), "drop_params" in data, proxy_config ) + token_fields: Final = _token_count_fields(data, auto_drop_params, llm_router, user_api_key_dict.team_id) + token_request: Final = ( + token_fields if isinstance(token_fields, ProxyException) else _token_count_request(token_fields) + ) + if isinstance(token_request, ProxyException): + return _anthropic_error_json_response(token_request, request) token_response: Final = await count_request_tokens( request=token_request, call_endpoint=True, messages_format="anthropic", - **_count_tokens_body_options( - data, - should_auto_drop_params_for_agentic_cli( - request.headers.get("user-agent", ""), "drop_params" in data, proxy_config - ), - ), + **_count_tokens_body_options(data, auto_drop_params), ) except HTTPException as e: return _anthropic_error_json_response(proxy_exception_from_http_exception(e, {}), request) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 5f798fce6cf..2d387b43aa8 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -80,6 +80,7 @@ from litellm.litellm_core_utils.served_output_texts import ( from litellm.litellm_core_utils.streaming_handler import ( backfill_missing_cache_usage_fields, ) +from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, @@ -129,6 +130,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports ) from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict +from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model from litellm.types.guardrails import GuardrailEventHooks from litellm.types.router import RouterRateLimitError @@ -633,6 +635,63 @@ def serialize_http_exception_detail( return str(detail), None +_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True)) +_UNSEEN_DEPLOYMENT: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset()) + + +def _is_auto_router(params: Mapping[str, object]) -> bool: + model: Final = params.get("model") + return isinstance(model, str) and model.startswith(AUTO_ROUTER_MODEL_PREFIX) + + +def _named_deployments(llm_router: Router, model: str, team_id: str | None) -> tuple[Mapping[str, object], ...]: + by_id: Final = None if model in llm_router.model_names else llm_router.get_deployment(model_id=model) + if by_id is not None: + return (_REQUEST_FIELDS.validate_python(by_id.litellm_params.model_dump(exclude_none=True)),) + group: Final = llm_router.get_model_from_alias(model=model) or model + if group not in llm_router.model_names or llm_router.get_model_list_from_routing_groups(model_name=group): + return () + deployments: Final = tuple( + _REQUEST_FIELDS.validate_python(deployment["litellm_params"]) + for deployment in llm_router.get_all_deployments(model_name=group, team_id=team_id) + ) + return () if any(_is_auto_router(params) for params in deployments) else deployments + + +def _router_defaults(llm_router: Router | None) -> Mapping[str, object]: + configured: Final[object] = getattr(llm_router, "default_litellm_params", None) + if configured is None: + return {} + return {name: value for name, value in _REQUEST_FIELDS.validate_python(configured).items() if value is not None} + + +def _counted_drop_policies( + data: Mapping[str, object], llm_router: Router | None, team_id: str | None +) -> tuple[DropPolicy, ...]: + requested: Final = {**_router_defaults(llm_router), **data} + model: Final = data.get("model") + deployments: Final = ( + _named_deployments(llm_router, model, team_id) if llm_router is not None and isinstance(model, str) else () + ) + if not deployments: + return (drop_policy(requested), _UNSEEN_DEPLOYMENT) + return tuple(drop_policy({**deployment, **requested}) for deployment in deployments) + + +def resolve_counted_tool_fields( + data: Mapping[str, object], llm_router: Router | None, team_id: str | None +) -> Mapping[str, object] | ProxyException: + resolved: Final = resolve_tool_fields(data, lambda: _counted_drop_policies(data, llm_router, team_id)) + if isinstance(resolved, MalformedField): + return ProxyException( + message=resolved.message, + type="invalid_request_error", + param=resolved.field, + code=status.HTTP_400_BAD_REQUEST, + ) + return resolved + + def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, str]) -> ProxyException: raw_detail: Final = attribute_of(exc, "detail", str(exc)) message, structured_fields = serialize_http_exception_detail(raw_detail) diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py index 320d687b3aa..242354ac417 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py @@ -32,6 +32,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" additional_provider_specific_params=getattr(litellm_params, "additional_provider_specific_params", {}), unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"), fail_on_error=getattr(litellm_params, "fail_on_error", True), + forward_messages_as_sent=getattr(litellm_params, "forward_messages_as_sent", False), extra_headers=getattr(litellm_params, "extra_headers", None), guardrail_name=guardrail.get("guardrail_name", ""), event_hook=litellm_params.mode, diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index 25df7a20e14..93b08831d84 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -7,10 +7,12 @@ import fnmatch import os -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, Optional import httpx +from pydantic import JsonValue, TypeAdapter, ValidationError +from pydantic_core import PydanticSerializationError from litellm._logging import verbose_proxy_logger from litellm._version import version as litellm_version @@ -38,6 +40,10 @@ if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel GUARDRAIL_NAME: Final = "generic_guardrail_api" +_CHAT_ROW: Final[TypeAdapter[AllMessageValues]] = TypeAdapter(AllMessageValues) +_SENT_ROW: Final = TypeAdapter(Mapping[str, object]) +_SENT_ROWS: Final = TypeAdapter(Sequence[Mapping[str, object]]) +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) # Headers whose values are forwarded as-is (case-insensitive). Glob patterns supported (e.g. x-stainless-*, x-litellm*). _HEADER_VALUE_ALLOWLIST: Final = frozenset( @@ -150,23 +156,65 @@ def _extract_inbound_headers( return None +def _is_contained_in(part: JsonValue, whole: JsonValue) -> bool: + pending: Final[list[tuple[JsonValue, JsonValue]]] = [(part, whole)] # mutable-ok: worklist, recursion is banned + while pending: + match pending.pop(): + case dict() as part_dict, dict() as whole_dict: + if not part_dict.keys() <= whole_dict.keys(): + return False + pending.extend((value, whole_dict[key]) for key, value in part_dict.items()) + case list() as part_list, list() as whole_list: + if len(part_list) != len(whole_list): + return False + pending.extend(zip(part_list, whole_list, strict=True)) + case part_leaf, whole_leaf if part_leaf != whole_leaf: + return False + case _: + pass + return True + + +def _is_shown_row(shown: AllMessageValues, returned: AllMessageValues) -> bool: + return returned == shown + + +def _is_echo(original: AllMessageValues, returned: AllMessageValues) -> bool: + """A server that parses rows with its own message types and echoes them back drops the + keys those types do not declare. That is still an echo as long as nothing changed and + every key the chat completion types declare is kept.""" + if returned == original: + return True + try: + sent: Final[JsonValue] = _JSON.validate_json(_SENT_ROW.dump_json(original)) + declared: Final[JsonValue] = _JSON.validate_json( + _CHAT_ROW.dump_json(_CHAT_ROW.validate_python(original), warnings="error") + ) + echoed: Final[JsonValue] = _JSON.validate_python(returned) + except (ValidationError, PydanticSerializationError): + return False + return _is_contained_in(echoed, sent) and _is_contained_in(declared, echoed) + + def _structured_rows_to_write_back( original_rows: Sequence[AllMessageValues] | None, - shown_rows: Sequence[AllMessageValues] | None, + compared_rows: Sequence[AllMessageValues] | None, returned_rows: Sequence[AllMessageValues], + is_echo: Callable[[AllMessageValues, AllMessageValues], bool], ) -> tuple[AllMessageValues, ...] | None: - """The request model drops row keys its message types do not declare, so a - row the server echoes back verbatim is restored to the original row object. - A server that echoes every row back unchanged has not rewritten anything - per row, so its answer is read from texts, as it was before rows could be - returned at all.""" - if original_rows is None or shown_rows is None or len(returned_rows) != len(original_rows): + """A row the server echoes back is restored to the original row object. A server + that echoes every row back has not rewritten anything per row, so its answer is + read from texts, as it was before rows could be returned at all.""" + if original_rows is None or compared_rows is None or len(returned_rows) != len(original_rows): return tuple(returned_rows) - if all(returned == shown for shown, returned in zip(shown_rows, returned_rows)): + echoed: Final = tuple( + is_echo(compared, returned) for compared, returned in zip(compared_rows, returned_rows, strict=True) + ) + if all(echoed): return None return tuple( - original if returned == shown else returned - for original, shown, returned in zip(original_rows, shown_rows, returned_rows) + original if was_echoed else returned + for original, returned, was_echoed in zip(original_rows, returned_rows, echoed, strict=True) ) @@ -200,6 +248,7 @@ class GenericGuardrailAPI(CustomGuardrail): additional_provider_specific_params: Mapping[str, object] | None = None, unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed", fail_on_error: bool | None = True, + forward_messages_as_sent: bool | None = False, extra_headers: list | None = None, streaming_end_of_stream_only: bool | None = None, streaming_sampling_rate: int | None = None, @@ -235,6 +284,7 @@ class GenericGuardrailAPI(CustomGuardrail): self.unreachable_fallback: Literal["fail_closed", "fail_open"] = unreachable_fallback self.fail_on_error: bool = True if fail_on_error is None else fail_on_error + self.forward_messages_as_sent: bool = bool(forward_messages_as_sent) # Read by UnifiedLLMGuardrails.async_post_call_streaming_iterator_hook # via getattr(guardrail_to_apply, "streaming_*", default). @@ -316,15 +366,17 @@ class GenericGuardrailAPI(CustomGuardrail): http_status_code: int | None = None, ) -> GenericGuardrailAPIInputs: status_suffix: Final = f" http_status_code={http_status_code}" if http_status_code else "" + error_type: Final = "HTTPStatusError" if isinstance(error, httpx.HTTPStatusError) else type(error).__name__ verbose_proxy_logger.critical( "Generic Guardrail API error (fail-open). Proceeding without guardrail.%s " - "guardrail_name=%s api_base=%s input_type=%s litellm_call_id=%s litellm_trace_id=%s", + "guardrail_name=%s api_base=%s input_type=%s litellm_call_id=%s litellm_trace_id=%s error_type=%s", status_suffix, getattr(self, "guardrail_name", None), getattr(self, "api_base", None), input_type, getattr(logging_obj, "litellm_call_id", None) if logging_obj else None, getattr(logging_obj, "litellm_trace_id", None) if logging_obj else None, + error_type, exc_info=error, ) # Keep flow going - treat as action=NONE (no modifications) @@ -339,10 +391,23 @@ class GenericGuardrailAPI(CustomGuardrail): headers.update(self.headers) return headers + def _request_body( + self, + guardrail_request: GenericGuardrailAPIRequest, + structured_messages: Sequence[AllMessageValues] | None, + ) -> Mapping[str, object]: + body: Final[Mapping[str, object]] = guardrail_request.model_dump(mode="json") + if not self.forward_messages_as_sent or structured_messages is None: + return body + return { + **body, + "structured_messages": _SENT_ROWS.dump_python(_SENT_ROWS.validate_python(structured_messages), mode="json"), + } + def _build_guardrail_return_inputs( self, *, - texts: list, + texts: list[str], images: list[str] | None, tools: list[ChatCompletionToolParam] | None, structured_messages: Sequence[AllMessageValues] | None, @@ -361,8 +426,13 @@ class GenericGuardrailAPI(CustomGuardrail): return_inputs["tools"] = guardrail_response.tools elif tools: return_inputs["tools"] = tools + compared_rows, is_echo = ( + (structured_messages, _is_echo) if self.forward_messages_as_sent else (shown_messages, _is_shown_row) + ) rows_to_write_back: Final = ( - _structured_rows_to_write_back(structured_messages, shown_messages, guardrail_response.structured_messages) + _structured_rows_to_write_back( + structured_messages, compared_rows, guardrail_response.structured_messages, is_echo + ) if guardrail_response.structured_messages else None ) @@ -465,7 +535,7 @@ class GenericGuardrailAPI(CustomGuardrail): litellm_version=litellm_version, images=images, tools=([GuardrailToolParam.model_validate(t) for t in tools] if tools else None), - structured_messages=structured_messages, + structured_messages=None if self.forward_messages_as_sent else structured_messages, tool_calls=tool_calls, additional_provider_specific_params=additional_params, input_type=input_type, @@ -478,7 +548,7 @@ class GenericGuardrailAPI(CustomGuardrail): # Use mode="json" to ensure all iterables are converted to lists response: Final = await self.async_handler.post( url=self.api_base, - json=guardrail_request.model_dump(mode="json"), + json=dict(self._request_body(guardrail_request, structured_messages)), headers=headers, timeout=self.timeout, ) diff --git a/litellm/proxy/guardrails/tool_name_extraction.py b/litellm/proxy/guardrails/tool_name_extraction.py index 72e85f35f7d..526f58dcd36 100644 --- a/litellm/proxy/guardrails/tool_name_extraction.py +++ b/litellm/proxy/guardrails/tool_name_extraction.py @@ -6,23 +6,23 @@ knowledge lives in one place. Uses guardrail translation handlers where availabl with standalone extractors for generate_content and MCP. """ +from collections.abc import Mapping from typing import Any, Final from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.llms import load_guardrail_translation_mappings +from litellm.llms.base_llm.guardrail_translation.utils import list_field from litellm.types.utils import CallTypes # Call types that have no guardrail translation handler; we use standalone extractors STANDALONE_EXTRACTORS: Final[dict[str, Any]] = {} -def _extract_generate_content_tool_names(data: dict) -> list[str]: +def _extract_generate_content_tool_names(data: Mapping[str, object]) -> list[str]: """Google generateContent: tools[].functionDeclarations[].name""" names: Final[list[str]] = [] - for tool in data.get("tools") or []: - if not isinstance(tool, dict): - continue - for decl in tool.get("functionDeclarations") or []: + for tool in list_field(data, "tools"): + for decl in list_field(tool, "functionDeclarations"): if isinstance(decl, dict) and decl.get("name"): names.append(str(decl["name"])) return names diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 63cc8f0e3ba..b563dbc5c4c 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -425,7 +425,7 @@ async def add_new_user_to_default_team( user_id: str, user_email: str | None, user_api_key_dict: UserAPIKeyAuth, - teams: list[str] | list[NewUserRequestTeam], + teams: Sequence[str | NewUserRequestTeam], prisma_client: "PrismaClient", ): tasks: Final[list[Awaitable[object]]] = [] diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 36071bfe51a..7460522f53e 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -28,7 +28,7 @@ from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid -from litellm.constants import LITELLM_PROXY_ADMIN_NAME +from litellm.constants import LITELLM_PROXY_ADMIN_NAME, TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.ptu_pricing import ( @@ -1881,7 +1881,7 @@ async def delete_team_models( """ deleted_model_ids: Final[list[str]] = [] transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db) - async with transaction_client.tx() as tx: + async with transaction_client.tx(timeout=datetime.timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS)) as tx: for team_id in team_ids: rows = await _get_team_deployments(team_id, prisma_client, table=tx.litellm_proxymodeltable) model_ids = [row.model_id for row in rows] diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index ba30ac8609b..0f40662ad83 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -17,7 +17,7 @@ import traceback from collections.abc import Iterable, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from itertools import chain from types import MappingProxyType from typing import ( @@ -42,6 +42,7 @@ from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.caching.redis_batch import active_request_redis_batches +from litellm.constants import TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS from litellm.integrations.prometheus import PrometheusLogger from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import ( @@ -4674,7 +4675,7 @@ async def delete_team( # or is still waiting on the lock, in which case its own re-read happens after this commits # and sees the row gone before it writes anything. delete_filter: Final[_TeamIdInFilter] = {"team_id": {"in": data.team_ids}} - async with prisma_client.tx() as tx: + async with prisma_client.tx(timeout=timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS)) as tx: for team_id in sorted(data.team_ids): await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id) await tx.litellm_teamtable.delete_many(where=delete_filter) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index e6d93aa6260..4cc849762aa 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -89,6 +89,7 @@ from litellm.proxy._types import ( Member, NewTeamRequest, NewUserRequest, + NewUserRequestTeam, NewUserResponse, ProxyErrorTypes, ProxyException, @@ -118,7 +119,11 @@ from litellm.proxy.common_utils.html_forms.jwt_display_template import ( from litellm.proxy.common_utils.html_forms.ui_login import build_ui_login_form from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler -from litellm.proxy.management_endpoints.internal_user_endpoints import new_user +from litellm.proxy.management_endpoints.internal_user_endpoints import ( + add_new_user_to_default_team, + check_if_default_team_set, + new_user, +) from litellm.proxy.management_endpoints.sso import CustomMicrosoftSSO, LiteLLMGoogleSSO, create_generic_sso_provider from litellm.proxy.management_endpoints.sso.id_jag_assertion_capture import ( id_jag_assertion_capture_gap, @@ -2825,6 +2830,52 @@ async def insert_sso_user( return response +DefaultTeamJoin: TypeAlias = Callable[ + [LiteLLM_UserTable, Sequence[str | NewUserRequestTeam], PrismaClient], Awaitable[None] +] + + +async def _join_default_teams_as_admin( + user: LiteLLM_UserTable, teams: Sequence[str | NewUserRequestTeam], prisma_client: PrismaClient +) -> None: + await add_new_user_to_default_team( + user_id=user.user_id, + user_email=user.user_email, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + teams=teams, + prisma_client=prisma_client, + ) + + +def _default_teams_not_joined(user: LiteLLM_UserTable) -> tuple[str | NewUserRequestTeam, ...]: + joined: Final = frozenset(user.teams) + return tuple( + team + for team in check_if_default_team_set() or () + if (team if isinstance(team, str) else team.team_id) not in joined + ) + + +async def _insert_sso_user_or_find_concurrent_insert( + insert: Awaitable[NewUserResponse], + user_id: str | None, + prisma_client: PrismaClient, + join_default_teams: DefaultTeamJoin = _join_default_teams_as_admin, +) -> NewUserResponse | LiteLLM_UserTable: + try: + return await insert + except Exception: + users: Final = UserRepository(prisma_client, use_writer=True) + concurrent_insert: Final = None if user_id is None else await users.find_by_id(user_id) + if concurrent_insert is None: + raise + unjoined: Final = _default_teams_not_joined(concurrent_insert) + if not unjoined: + return concurrent_insert + await join_default_teams(concurrent_insert, unjoined, prisma_client) + return await users.find_by_id(concurrent_insert.user_id) or concurrent_insert + + @router.get( "/sso/get/ui_settings", tags=["experimental"], @@ -3471,9 +3522,10 @@ class SSOAuthenticationHandler: else: verbose_proxy_logger.info("user not in DB, inserting user into LiteLLM DB") # user not in DB, insert User into LiteLLM DB - user_info = await insert_sso_user( - result_openid=result, - user_defined_values=user_defined_values, + user_info = await _insert_sso_user_or_find_concurrent_insert( + insert=insert_sso_user(result_openid=result, user_defined_values=user_defined_values), + user_id=None if user_defined_values is None else user_defined_values.get("user_id"), + prisma_client=prisma_client, ) return user_info except Exception as e: diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 23ef89b3c5c..a7f3a4f84a7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -10,12 +10,14 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ANTHROPIC_BATCHES_ROUTE +from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, ) from litellm.llms.anthropic import get_anthropic_config +from litellm.llms.anthropic.common_utils import anthropic_error_frame_exception from litellm.llms.anthropic.pass_through.stream_assembly import ( build_complete_streaming_response, build_complete_streaming_response_legacy, @@ -254,6 +256,26 @@ class AnthropicPassthroughLoggingHandler: ), ) + @staticmethod + async def log_error_frame_as_failure( + attempt_logging_obj: LiteLLMLoggingObj, + request_body: Mapping[str, object], + all_chunks: Sequence[str | bytes], + error_event: tuple[str, str, int], + ) -> None: + """A stream closed at the provider's `event: error` frame is one failed attempt of the request: it logs + through the attempt-scoped copy its caller took at the close, so the usage it consumed never reaches + the success log of the attempt the router opens in its place.""" + error_type, message, status_code = error_event + await asyncify(AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure)( + litellm_logging_obj=attempt_logging_obj, request_body=request_body, all_chunks=all_chunks + ) + await attempt_logging_obj.dispatch_failure_handlers( + anthropic_error_frame_exception(error_type, message, status_code, str(attempt_logging_obj.model)), + "", + prefer_async_handlers=True, + ) + @staticmethod def _cost_partial_stream_or_zero( partial_response: ModelResponse | TextCompletionResponse, model: str, logging_obj: LiteLLMLoggingObj diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 8f6a688fecd..6799b82bf06 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -165,6 +165,7 @@ class PassThroughStreamingHandler: ) logging_scheduled = False + stream_ended = False # rebind-ok: flipped once the upstream ran to its end, so the finally can tell a close-at-frame from a completed stream model_name: Final = PassThroughStreamingHandler._extract_model_for_cost_injection( request_body=request_body, url_route=url_route, @@ -224,6 +225,7 @@ class PassThroughStreamingHandler: ) if pending: yield pending + stream_ended = True # rebind-ok: the finally reads it to spot a close-at-frame # Stream completed cleanly. When the proxy armed deferred # dispatch (post-call guardrails active), park the logging # coroutine on logging_obj instead of enqueueing now, so @@ -267,13 +269,46 @@ class PassThroughStreamingHandler: # a success would double-log the same request. if not logging_scheduled and raw_bytes and response.status_code < 400: logging_scheduled = True + superseded_error_event: Final = ( + None + if stream_ended + else PassThroughStreamingHandler._error_frame_closed_at(endpoint_type, raw_bytes) + ) try: - GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=_build_logging_coroutine()) + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( + async_coroutine=( + _build_logging_coroutine() + if superseded_error_event is None + else AnthropicPassthroughLoggingHandler.log_error_frame_as_failure( + attempt_logging_obj=litellm_logging_obj.attempt_scoped_copy(), + request_body=resolved_request_body, + all_chunks=tuple(raw_bytes), + error_event=superseded_error_event, + ) + ) + ) except Exception as e: verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e) else: bind_budget_reservation_to_callbacks(litellm_logging_obj.litellm_params) + @staticmethod + def _error_frame_closed_at(endpoint_type: EndpointType, raw_bytes: Sequence[bytes]) -> tuple[str, str, int] | None: + """The provider `event: error` frame of a stream the router closed because a retry or fallback took the + request over. That attempt is logged as its own failure, on a copy taken now so the retry's rebinding of + the shared logging object never reaches it. A stream that ran to its end or that the client closed keeps + logging through the success route, so its usage is still billed.""" + from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + closing_superseded_attempt, + parse_anthropic_error_event, + ) + + if endpoint_type != EndpointType.ANTHROPIC or not closing_superseded_attempt(): + return None + + complete_frames, _ = split_complete_sse_frames(b"".join(raw_bytes)) + return parse_anthropic_error_event(complete_frames) + @staticmethod async def route_streaming_logging_to_handler( litellm_logging_obj: LiteLLMLoggingObj, diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index a44f1d10fda..207e966ef28 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -34,6 +34,7 @@ from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup +from litellm.litellm_core_utils.tool_fields import first_validation_error from litellm.llms.openai.responses.guardrail_translation.handler import ( OpenAIResponsesHandler, build_blocked_response, @@ -45,7 +46,12 @@ from litellm.proxy.auth.user_api_key_auth import ( user_api_key_auth, user_api_key_auth_websocket, ) -from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing, cost_headers, create_response +from litellm.proxy.common_request_processing import ( + ProxyBaseLLMRequestProcessing, + cost_headers, + create_response, + resolve_counted_tool_fields, +) from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _safe_set_request_parsed_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -57,6 +63,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio request_tags_from_request_data, ) from litellm.proxy.route_llm_request import raise_if_required_body_param_missing +from litellm.router import Router from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( REASONING_EFFORT, @@ -69,7 +76,6 @@ from litellm.types.utils import TokenCountResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.utils import ProxyLogging - from litellm.router import Router router: Final = APIRouter() _RESPONSES_WS_CONFIG_VALUE_ADAPTER: Final[TypeAdapter[object | None]] = TypeAdapter(object | None) @@ -1376,6 +1382,15 @@ def _proxy_token_counter() -> _TokenCounter: _token_counter_dep: Final = Depends(_proxy_token_counter) +def _proxy_llm_router() -> Router | None: + from litellm.proxy.proxy_server import llm_router + + return llm_router + + +_llm_router_dep: Final = Depends(_proxy_llm_router) + + def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse: body: Final[_ResponsesApiErrorBody] = { "error": { @@ -1432,6 +1447,8 @@ def _responses_input_as_token_count_messages( async def responses_input_tokens( request: Request, token_counter: _TokenCounter = _token_counter_dep, + llm_router: Router | None = _llm_router_dep, + user_api_key_dict: UserAPIKeyAuth = _user_api_key_auth_dep, ): """ Count the input tokens of a Responses API request without calling the model. @@ -1464,6 +1481,13 @@ async def responses_input_tokens( code="missing_required_parameter", ) + counted_fields: Final = { + name: data[name] for name in ("model", "tools", "drop_params", "additional_drop_params") if name in data + } + token_fields: Final = resolve_counted_tool_fields(counted_fields, llm_router, user_api_key_dict.team_id) + if isinstance(token_fields, ProxyException): + return _responses_invalid_request_response(message=token_fields.message, param=token_fields.param, code=None) + try: payload: Final[_TokenCountPayload] = { "model": model_name, @@ -1471,9 +1495,13 @@ async def responses_input_tokens( input_value=input_value, instructions=data.get("instructions"), ), - "tools": data.get("tools"), + "tools": token_fields.get("tools"), } token_request: Final = TokenCountRequest.model_validate(payload) + except ValidationError as e: + return _responses_invalid_request_response( + message=first_validation_error(e), param=str(e.errors(include_url=False)[0]["loc"][0]), code=None + ) except Exception as e: return _responses_invalid_request_response( message=f"Invalid request for token counting: {e}", param=None, code=None diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index dc6423c979b..36ace3c1697 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3493,6 +3493,7 @@ class ProxyLogging: Is triggered when self._is_proxy_only_error() returns True """ litellm_logging_obj: Logging | None = request_data.get("litellm_logging_obj", None) + set_up_by_request: Final = litellm_logging_obj is not None if litellm_logging_obj is None: from litellm._uuid import uuid @@ -3557,7 +3558,9 @@ class ProxyLogging: input = request_data["input"] litellm_logging_obj.model_call_details["input"] = input body_shape_call_type = CallTypes.aembedding.value - resolved_call_type: Final = _call_type_for_route(route) or body_shape_call_type + resolved_call_type: Final = _call_type_for_route(route) or ( + None if set_up_by_request else body_shape_call_type + ) if resolved_call_type is not None and litellm_logging_obj.call_type != CallTypes.pass_through.value: litellm_logging_obj.call_type = resolved_call_type litellm_logging_obj.model_call_details["call_type"] = resolved_call_type diff --git a/litellm/router.py b/litellm/router.py index 4f2358f9008..d041f958e17 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -123,6 +123,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( mask_sensitive_structure, ) from litellm.litellm_core_utils.token_counter import offload_token_count +from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy from litellm.llms.anthropic.common_utils import AnthropicModelInfo, anthropic_error_frame_exception from litellm.llms.base_llm.passthrough.transformation import replace_path_segment from litellm.llms.base_llm.vector_store.transformation import ( @@ -537,6 +538,7 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({}) +_NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -573,6 +575,21 @@ def _deployment_that_just_failed(request_metadata: object) -> str | None: return model_id if isinstance(model_id, str) else None +def _drops_malformed_request_tools( + deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object] +) -> bool: + if isinstance(request.get("tools") or [], (list, tuple)) or not deployment_params.get("tools"): + return False + set_defaults: Final = {name: value for name, value in defaults.items() if value is not None} + return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools") + + +def _merged_tools(deployment_tools: Sequence[object], request_tools: object) -> list[object] | None: + if not isinstance(request_tools, (list, tuple)): + return None + return [*deployment_tools, *request_tools] if deployment_tools or request_tools else None + + def _deployment_pick_attributes(model: str, request_kwargs: Mapping[str, object] | None) -> Mapping[str, str | int]: """Bounded attributes for one deployment pick; attempt is 1-based within the current model group.""" kwargs: Final = request_kwargs or {} @@ -4507,7 +4524,9 @@ class Router: return deployment_pydantic_obj @staticmethod - def _merge_tools_from_deployment(deployment: dict, kwargs: dict) -> None: + def _merge_tools_from_deployment( + deployment: dict, kwargs: dict, defaults: Mapping[str, object] = _NO_DEFAULT_PARAMS + ) -> None: """ Merge tools from deployment litellm_params with request kwargs. When both have tools, concatenate them (deployment tools first, then request tools). @@ -4518,11 +4537,12 @@ class Router: dep_params = dep_params_raw else: dep_params = dep_params_raw.model_dump(exclude_none=True) - dep_tools: Final = dep_params.get("tools") or [] - req_tools: Final = kwargs.get("tools") or [] - if dep_tools or req_tools: - merged: Final = list(dep_tools) + list(req_tools) - kwargs["tools"] = merged + if _drops_malformed_request_tools(dep_params, defaults, kwargs): + for name in TOOL_FIELDS: + kwargs.pop(name, None) + merged_tools: Final = _merged_tools(dep_params.get("tools") or [], kwargs.get("tools") or []) + if merged_tools is not None: + kwargs["tools"] = merged_tools if "tool_choice" not in kwargs and dep_params.get("tool_choice") is not None: kwargs["tool_choice"] = dep_params["tool_choice"] @@ -4543,7 +4563,7 @@ class Router: deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ()) ): kwargs.pop(key, None) - self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs) + self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=self.default_litellm_params) model_info = deployment.get("model_info", {}).copy() deployment_litellm_model_name = deployment["litellm_params"]["model"] @@ -6035,6 +6055,7 @@ class Router: """ from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( aclose_if_supported, + aclose_superseded_attempt, parse_anthropic_error_event, parse_anthropic_refusal_stop_details, ) @@ -6044,6 +6065,14 @@ class Router: source_iterator: Final = response + async def close_source() -> None: + with anyio.CancelScope(shield=True), contextlib.suppress(BaseException): + await aclose_if_supported(source_iterator) + + async def close_superseded() -> None: + with anyio.CancelScope(shield=True), contextlib.suppress(BaseException): + await aclose_superseded_attempt(source_iterator) + async def stream_with_fallbacks() -> AsyncGenerator[bytes, None]: from litellm.exceptions import MidStreamFallbackError @@ -6130,11 +6159,11 @@ class Router: model, initial_kwargs, wrapper, + close_superseded, ): yield item finally: - with anyio.CancelScope(shield=True), contextlib.suppress(BaseException): - await aclose_if_supported(source_iterator) + await close_source() # Referenced by stream_with_fallbacks via closure - assigned here, before # the generator body ever runs, so the reference resolves fine despite @@ -6150,6 +6179,7 @@ class Router: model: str, initial_kwargs: dict[str, Any], # mutable-ok: handed to _aanthropic_messages_fallback_attempt, which mutates it wrapper: "FallbackAwareAnthropicMessagesStream", + close_superseded: Callable[[], Awaitable[None]], ) -> AsyncGenerator[bytes, None]: """Turns a source-iterator failure into a fallback attempt or the error reaching the caller.""" from litellm.exceptions import MidStreamFallbackError @@ -6169,6 +6199,7 @@ class Router: ) if fallback_error is None: raise stream_error + await close_superseded() outcome: Final = await self._astream_retry_same_group( fallback_error, initial_kwargs, diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 316d76e67b1..e2ebb773db2 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -364,10 +364,10 @@ def filter_web_search_deployments( if isinstance(healthy_deployments, dict): return healthy_deployments - tools_value: Final[object] = request_kwargs.get("tools") or [] - tools: Final = _OBJECT_TUPLE_ADAPTER.validate_python(tools_value) - is_web_search_request: Final = any(_is_web_search_tool(tool) for tool in tools) - + tools: Final[object] = request_kwargs.get("tools") + is_web_search_request: Final = isinstance(tools, (list, tuple)) and any( + _is_web_search_tool(tool) for tool in _OBJECT_TUPLE_ADAPTER.validate_python(tools) + ) if not is_web_search_request: return healthy_deployments diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index c7e3dab0d1a..bb64d912c4c 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -329,6 +329,12 @@ def _should_run_cooldown_logic( ) return False + if isinstance(original_exception, litellm.MalformedRequestFieldError): + verbose_router_logger.debug( + "Should Not Run Cooldown Logic: litellm rejected the request before calling the deployment" + ) + return False + if request_kwargs is not None and _is_per_user_provider_request(request_kwargs): verbose_router_logger.debug( "Should Not Run Cooldown Logic: the failure came from one caller's per-user provider session" diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index f20a3fa3d7c..33db8cbc6f0 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -1195,6 +1195,16 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up description="Maximum decisions calls in flight at once on the guardrail in each proxy worker. Implemented by guardrail='decision_model'.", ) + forward_messages_as_sent: bool | None = Field( + default=False, + description=( + "Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the " + "guardrail exactly as the client sent it, so rows can carry provider-native content blocks and " + "keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with " + "undeclared keys removed." + ), + ) + extra_headers: list[str] | None = Field( default=None, description=( diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py index 49eaa154fec..d34a3a13838 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/generic_guardrail_api.py @@ -1,7 +1,7 @@ from collections.abc import Mapping, Sequence from typing import Any, Final, Literal, cast # noqa: TID251 # JSON chat rows have no typed constructor across roles -from pydantic import ConfigDict, Field +from pydantic import ConfigDict, Field, SerializerFunctionWrapHandler, model_serializer from typing_extensions import TypedDict from litellm.types.llms.base import LiteLLMBaseModel @@ -17,12 +17,20 @@ class GuardrailToolParam(LiteLLMBaseModel): """A tool forwarded verbatim to the guardrail for inspection. Built-in tools (code_interpreter, file_search, ...) have no ``function`` block - and stash their config in tool-specific keys, so only ``type`` is required and - ``extra="allow"`` preserves the rest instead of stripping it. + and stash their config in tool-specific keys, and provider-native tools such as + Gemini ``function_declarations`` carry no ``type`` at all, so nothing is required + and ``extra="allow"`` preserves the rest instead of stripping it. """ model_config = ConfigDict(extra="allow") - type: str + type: str | None = None + + @model_serializer(mode="wrap") + def _without_absent_type(self, handler: SerializerFunctionWrapHandler) -> Mapping[str, object]: + dumped: Final = cast("Mapping[str, object]", handler(self)) # cast-ok: handler dumps this model's fields + if self.type is not None: + return dumped + return {key: value for key, value in dumped.items() if key != "type"} class GenericGuardrailAPIMetadata(TypedDict, total=False): @@ -62,6 +70,15 @@ class GenericGuardrailAPIOptionalParams(LiteLLMBaseModel): ), ) + forward_messages_as_sent: bool | None = Field( + default=False, + description=( + "When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can " + "carry provider-native content blocks and keys the OpenAI chat types do not declare. False " + "(default) sends OpenAI-format chat rows with undeclared keys removed." + ), + ) + streaming_end_of_stream_only: bool | None = Field( default=None, description=( diff --git a/litellm/utils.py b/litellm/utils.py index 5a933d8f2b8..76ef50c3a2b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -92,6 +92,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.litellm_core_utils.tokenizer import strip_special_tokens +from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.rust_bridge.catalog import decision from litellm.rust_bridge.configuration import Decision @@ -474,6 +475,7 @@ from .exceptions import ( BudgetExceededError, ContentPolicyViolationError, ContextWindowExceededError, + MalformedRequestFieldError, ModelNotMappedError, NotFoundError, OpenAIError, @@ -1495,7 +1497,23 @@ def check_coroutine(value) -> bool: return get_coroutine_checker().is_async_callable(value) -async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str): +def _deployment_provider(kwargs: Mapping[str, object]) -> str: + model: Final = kwargs.get("model") + custom_llm_provider: Final = kwargs.get("custom_llm_provider") + api_base: Final = kwargs.get("api_base") + if not isinstance(model, str): + return "" + try: + return litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + api_base=api_base if isinstance(api_base, str) else None, + )[1] + except BadRequestError: + return "" + + +async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str) -> dict[str, Any]: """ Allow modifying the request just before it's sent to the deployment. @@ -1506,7 +1524,16 @@ async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str) except ValueError: typed_call_type = None # unknown call type - modified_kwargs = kwargs.copy() + route: Final[object] = getattr(kwargs.get("litellm_logging_obj"), "call_type", None) + resolved: Final = resolve_deployment_tool_fields(kwargs, route) + if isinstance(resolved, MalformedField): + raise MalformedRequestFieldError( + message=resolved.message, + model=kwargs.get("model"), + llm_provider=_deployment_provider(kwargs), + body={"type": "invalid_request_error", "param": resolved.field}, + ) + modified_kwargs = dict(resolved) custom_logger_class: Final = _get_cached_custom_logger() for callback in litellm.callbacks: diff --git a/litellm/vector_stores/vector_store_registry.py b/litellm/vector_stores/vector_store_registry.py index e0b536d6c57..b93d9874dbc 100644 --- a/litellm/vector_stores/vector_store_registry.py +++ b/litellm/vector_stores/vector_store_registry.py @@ -1,6 +1,6 @@ # litellm/proxy/vector_stores/vector_store_registry.py import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime, timezone from itertools import chain from typing import ( @@ -11,7 +11,7 @@ from typing import ( get_args, ) -from pydantic import ConfigDict, TypeAdapter +from pydantic import ConfigDict, TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import remove_items_at_indices @@ -34,6 +34,26 @@ else: PrismaClient = Any _DB_ROW: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) +_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True)) +_TOOL: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True)) +_STORE_ID_ITEMS: Final = TypeAdapter(list[object], config=ConfigDict(strict=True)) + + +def _named_store_ids(store_ids: object) -> tuple[str, ...]: + if isinstance(store_ids, str): + return (store_ids,) + try: + items: Final = _STORE_ID_ITEMS.validate_python(store_ids) + except ValidationError: + return () + return tuple(store_id for store_id in items if isinstance(store_id, str)) + + +def _tool_vector_store_ids(tool: object) -> tuple[str, ...]: + try: + return _named_store_ids(_TOOL.validate_python(tool).get("vector_store_ids")) + except ValidationError: + return () class VectorStoreIndexRegistry: @@ -131,7 +151,7 @@ class VectorStoreRegistry: return VectorStoreToolParams(**kwargs) - def get_vector_store_ids_to_run(self, non_default_params: dict, tools: list[dict] | None = None) -> list[str]: + def get_vector_store_ids_to_run(self, non_default_params: dict, tools: object = None) -> list[str]: """ Returns the vector store ids to run @@ -175,7 +195,7 @@ class VectorStoreRegistry: tools_to_remove: Final[list[int]] = [] for i, tool in enumerate(tools): - tool_vector_store_ids = tool.get("vector_store_ids", []) + tool_vector_store_ids = _tool_vector_store_ids(tool) if not tool_vector_store_ids: continue @@ -395,14 +415,15 @@ class VectorStoreRegistry: return vector_stores_to_run - def _get_vector_store_ids_from_tool_calls(self, tools: list[dict] | None, vector_store_ids: list[str]) -> list[str]: + def _get_vector_store_ids_from_tool_calls(self, tools: object, vector_store_ids: list[str]) -> list[str]: """ Returns the vector store ids from the tool calls """ - return [ - *vector_store_ids, - *chain.from_iterable(tool["vector_store_ids"] for tool in tools or () if "vector_store_ids" in tool), - ] + try: + validated_tools: Final = _ITEMS.validate_python(tools) + except ValidationError: + return vector_store_ids + return [*vector_store_ids, *chain.from_iterable(_tool_vector_store_ids(tool) for tool in validated_tools)] def load_vector_stores_from_config(self, vector_stores_config: list[dict]): """ diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 6762e4cd35c..e669107900d 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -112,6 +112,8 @@ ignored_function_names = [ "_get_wildcard_deployments", # Tested through the get_model_list_of_routed_group wildcard test in test_router.py "_is_fallback_hop", # Tested through the order fallback hop tests in test_router_order_fallback.py "_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py + "_merged_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py + "_drops_malformed_request_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py "_collect_ptu_share_teams", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py "_resolve_team_ptu_ceiling", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py "_register_continuation_prefill_check", # Tested through the continuation tests in test_router.py diff --git a/tests/integration/AGENTS.md b/tests/integration/AGENTS.md index 5a09b66e7ed..a727b20596c 100644 --- a/tests/integration/AGENTS.md +++ b/tests/integration/AGENTS.md @@ -25,3 +25,15 @@ By the domain a user would name: `pricing`, `spend`, `routing`, `mcp`. A file on directory that a `GROUPS` entry in `run.py` selects. There is no manifest and no `covers` marker on new tests. A product bug the test exposes is `pytest.skip("BUG: ")` at the top of the body, not a fix in the test and not a deletion. Needs no network, proxy, DB or Redis: `tests/unit` + +CircleCI's integration groups run every file here except the OCI and sandbox tests, which no CI runs, and +the files in `GITHUB_FILES` in `run.py`. CircleCI runs on main, not on pull requests, so a new file here +first runs after its PR merges. Wanting a file to run before merge is no reason to change CI, and a new +dedicated workflow for it is the same mistake as a shared shard. Write the file, leave `GITHUB_FILES` and +every workflow alone, and tell the user it first runs on main. `GITHUB_FILES` is only for a file that +needs what CircleCI's groups cannot give it, like the images `image-scan.yml` builds or the plain pytest +run CircleCI's `roi-database` job gives `test_roi_observed.py`. Add a file there only in that case, +together with its dedicated job, and list a new dedicated GitHub workflow with the directory it owns in +`DEDICATED_INTEGRATION_WORKFLOWS` in `.github/scripts/assert_ci_coverage.py`. The `ci-coverage` check +fails a file here that a GitHub workflow runs unless it is in `GITHUB_FILES` and that workflow owns its +directory (#46780) diff --git a/tests/integration/README.md b/tests/integration/README.md index 33263043489..496c88050ee 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -4,7 +4,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport -The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so they run in the GitHub Actions `redis-compat` matrix against every supported redis-py major rather than in `run_integration.sh` +The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so CircleCI's `sdk` group installs `redis-server` before it runs them The `cost` group is driven by the `CostTrackingTestCase` literals under `cost_calculation/cases/.py` (gathered by `cost_calculation/catalog.py`), with the scripted cost map in `cost_calculation/cost_map.py`. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL` @@ -26,7 +26,7 @@ Reuse the existing canned provider handlers through `_support/upstream.py`. It r Owned proxies disable `.env` loading by default. A contract testing dotenv behavior can explicitly set `PYTHON_DOTENV_DISABLED=0` in its owned environment. The runner drops inherited `*_API_KEY` variables, so a contract that needs one sets a synthetic value in its owned environment. An explicit scratch writer database drops an inherited reader with a different database name; pass both database URLs when the contract needs a paired reader. Database relays close listeners, active connections and their event loops on teardown. Upstream startup failures reap the child process before returning the failure. Custom subprocesses, direct SQL and background tasks still need an explicit owner in the test -Owned proxy workers configure a 32-thread executor before application startup, preserving additional worker startup hooks. Held synchronous provider calls must not consume a CPU-dependent executor limit before the rest of a fault burst can start. The proxy entrypoint imports LiteLLM inside `main`, so spawned workers begin supervision before importing the application. Worker healthchecks have a five-second deadline by default, so uvicorn kills a worker that stays paused longer than that. Tests never signal proxy processes themselves. `OwnedProxy.kill_worker` SIGKILLs a worker and returns once its replacement answers readiness with its own pid, `replacing_worker` does the same for a kill that has to land mid-burst, and `suspended` pauses a worker or the supervisor, which only works on a proxy launched with `healthcheck=WorkerHealthcheck.LENIENT`. Helper processes that are not the proxy stop through `stop_child`, `stop_processes` and `stop_process_group` and pause through `paused`. `scripts/check_test_quality.py` rule TQ010 rejects a raw `os.kill`, SIGKILL / SIGSTOP / SIGCONT, `.kill()`, `.terminate()`, `.suspend()` or `.send_signal()` anywhere under `tests/integration` outside `_support`, and TQ011 rejects an assertion that caps an elapsed time with a fixed number. Bound timings with `_support/timing.py` instead, relative to a measured control or to the injected delay. `run.py` measures how long this box takes to import the proxy application before it starts pytest, records it in `boot-calibration.json` beside the results and exports it as `INTEGRATION_IMPORT_SECONDS`. Readiness deadlines, graceful stops and the default per-test timeout are multiples of that measurement (`_support/budget.py`), so a slower box gets longer budgets and a two-worker proxy gets twice the readiness deadline of a one-worker one. Readiness requires responsive worker identities that are still alive and a positively identified owned listener. Unreadable socket records cannot establish ownership. Crash tests own their client tasks with `TaskGroup`, release held peers in `finally` and verify that the intended worker dies. A test requiring traffic on both workers prepares that traffic explicitly instead of assuming OS load balancing, for example by sending each half while the other worker is suspended +Owned proxy workers configure a 32-thread executor before application startup, preserving additional worker startup hooks. Held synchronous provider calls must not consume a CPU-dependent executor limit before the rest of a fault burst can start. The proxy entrypoint imports LiteLLM inside `main`, so spawned workers begin supervision before importing the application. Worker healthchecks have a five-second deadline by default, so uvicorn kills a worker that stays paused longer than that. Tests never signal proxy processes themselves. `OwnedProxy.kill_worker` SIGKILLs a worker and returns once its replacement answers readiness with its own pid, `replacing_worker` does the same for a kill that has to land mid-burst, and `suspended` pauses a worker or the supervisor, which only works on a proxy launched with `healthcheck=WorkerHealthcheck.LENIENT`. The `kill()` that `replacing_worker` yields returns as soon as the worker is dead, so a test can release requests it holds on the surviving worker right away. Uvicorn may take a full worker healthcheck to notice the death, so the block waits for the supervisor to reap the worker and for the replacement only when it exits. Helper processes that are not the proxy stop through `stop_child`, `stop_processes` and `stop_process_group` and pause through `paused`. `scripts/check_test_quality.py` rule TQ010 rejects a raw `os.kill`, SIGKILL / SIGSTOP / SIGCONT, `.kill()`, `.terminate()`, `.suspend()` or `.send_signal()` anywhere under `tests/integration` outside `_support`, and TQ011 rejects an assertion that caps an elapsed time with a fixed number. Bound timings with `_support/timing.py` instead, relative to a measured control or to the injected delay. `run.py` measures how long this box takes to import the proxy application before it starts pytest, records it in `boot-calibration.json` beside the results and exports it as `INTEGRATION_IMPORT_SECONDS`. Readiness deadlines, graceful stops and the default per-test timeout are multiples of that measurement (`_support/budget.py`), so a slower box gets longer budgets and a two-worker proxy gets twice the readiness deadline of a one-worker one. Readiness requires responsive worker identities that are still alive and a positively identified owned listener. Unreadable socket records cannot establish ownership. Crash tests own their client tasks with `TaskGroup`, release held peers in `finally` and verify that the intended worker dies. A test requiring traffic on both workers prepares that traffic explicitly instead of assuming OS load balancing, for example by sending each half while the other worker is suspended Any port the harness picks before the server that listens on it has started comes from `allocate_port()` in `_support/ports.py`. That covers owned proxies, scripted upstreams, fake providers, OTLP sinks, database relays and Redis. A server that binds port 0 itself and keeps that socket, such as `wire_server` without a port, needs no allocation. Each pytest process claims a block of 200 ports between 10000 and 32767 by holding an exclusive lock file per block in the system temp directory, so two runs on one machine, such as parallel worktrees, never share a block, and the OS drops the lock when the process exits. The range sits above the fixed service ports and below the Linux and macOS ephemeral ranges, so a port-0 bind or an outbound connection never lands in a block. Ports inside a block go out in order and are never handed out twice in a run, a port that something outside the harness already holds is skipped, and a process that uses up its block claims the next free one. A test that restarts a server on the same port allocates that port up front instead of reading it back from the first server. TQ012 in `scripts/check_test_quality.py` rejects reading a port back from a port-0 bind anywhere under `tests/integration` diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 3c12414553d..fd61ca3aea2 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -150,9 +150,9 @@ def _is_reaped(process: psutil.Process) -> bool: return False -def _wait_until_reaped(process: psutil.Process, supervisor: psutil.Process, seconds: float) -> None: +def _wait_until_reaped(process: psutil.Process, seconds: float) -> None: deadline: Final = time.monotonic() + seconds - while not _is_reaped(process) and not _is_stopped(supervisor): + while not _is_reaped(process): assert time.monotonic() < deadline, f"The supervisor did not reap worker {process.pid} within {seconds:.0f}s" time.sleep(0.05) @@ -160,8 +160,6 @@ def _wait_until_reaped(process: psutil.Process, supervisor: psutil.Process, seco @dataclass(frozen=True, slots=True) class WorkerKill: victim: psutil.Process - supervisor: psutil.Process - reap_seconds: float fired: threading.Event def freeze(self) -> psutil.Process: @@ -173,7 +171,6 @@ class WorkerKill: with suppress(psutil.NoSuchProcess): self.victim.kill() _wait_until_gone(self.victim, STRICT_HEALTHCHECK_SECONDS) - _wait_until_reaped(self.victim, self.supervisor, self.reap_seconds) self.fired.set() @@ -250,22 +247,22 @@ class OwnedProxy: assert pid in before, f"{pid} is not a serving worker of this proxy: {sorted(before)}" return psutil.Process(pid), before - def _worker_kill(self, victim: psutil.Process) -> WorkerKill: - reap_seconds: Final = healthcheck_seconds(self.healthcheck) + STRICT_HEALTHCHECK_SECONDS - return WorkerKill(victim, psutil.Process(self.process.pid), reap_seconds, threading.Event()) + def _replacement_of(self, victim: psutil.Process, before: frozenset[int]) -> int: + _wait_until_reaped(victim, healthcheck_seconds(self.healthcheck) + STRICT_HEALTHCHECK_SECONDS) + return self._await_replacement(before) @contextmanager def replacing_worker(self, pid: int) -> Iterator[WorkerKill]: victim, before = self._victim(pid) - kill: Final = self._worker_kill(victim) + kill: Final = WorkerKill(victim, threading.Event()) yield kill assert kill.fired.is_set(), f"replacing_worker({pid}) exited without killing the worker" - self._await_replacement(before) + self._replacement_of(victim, before) def kill_worker(self, pid: int) -> int: victim, before = self._victim(pid) - self._worker_kill(victim)() - return self._await_replacement(before) + WorkerKill(victim, threading.Event())() + return self._replacement_of(victim, before) @contextmanager def suspended(self, pid: int) -> Iterator[psutil.Process]: diff --git a/tests/integration/management/test_team_delete_large_membership.py b/tests/integration/management/test_team_delete_large_membership.py index 90d204722ac..3e4bb568394 100644 --- a/tests/integration/management/test_team_delete_large_membership.py +++ b/tests/integration/management/test_team_delete_large_membership.py @@ -53,6 +53,13 @@ SELECT encode(sha256(convert_to(%s || '-' || n::text, 'UTF8')), 'hex'), %s, '{}' FROM generate_series(1, %s::int) AS n """ TAKE_TEAM_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext(%s))" +LOCK_MODEL_ROW_SQL: Final = 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s FOR UPDATE' +LOCK_HELD_PAST_DEFAULT_TRANSACTION_SECONDS: Final = 7 +OLDEST_TRANSACTION_BLOCKED_BY_SQL: Final = """ +SELECT coalesce(max(extract(epoch FROM clock_timestamp() - xact_start)), 0)::float AS age +FROM pg_stat_activity +WHERE %s::int = ANY(pg_blocking_pids(pid)) +""" # Sessions blocked on the advisory lock the given backend holds, and nothing else on the shared rig. WAITERS_ON_HELD_LOCK_SQL: Final = """ SELECT count(*)::int AS waiting @@ -131,6 +138,13 @@ def _waiters_on_lock_held_by(backend_pid: int) -> int: return waiting +def _oldest_transaction_blocked_by(backend_pid: int) -> float: + rows: Final = read_rows(OLDEST_TRANSACTION_BLOCKED_BY_SQL, (str(backend_pid),)) + age: Final = rows[0]["age"] + assert isinstance(age, float) + return age + + def _remove_team_by_sql(team_id: str) -> None: """Cleanup for a team the test expects to have deleted itself. Whatever a failed delete left behind (row, memberships, `teams` references) goes by SQL so the shared rig stays clean without sending @@ -161,6 +175,38 @@ def _create_team(scenario: Scenario, **fields: JsonValue) -> str: return team_id +def _create_team_model(scenario: Scenario, team_id: str) -> str: + created: Final = scenario.gateway.post( + "/model/new", + { + "model_name": f"integration-{uuid.uuid4().hex}", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-integration"}, + "model_info": {"team_id": team_id}, + }, + ) + model_id: Final = string_value(object_value(created["model_info"])["id"]) + scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (model_id,)) + return model_id + + +def _team_model_rows(model_id: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (model_id,)) + + +def _delete_while_locked(gateway: Gateway, team_id: str, lock_sql: str, lock_target: str) -> httpx.Response: + with ThreadPoolExecutor(max_workers=1) as pool: + with psycopg.connect(os.environ["DATABASE_URL"]) as holder: + holder.execute(lock_sql, (lock_target,)) + pending: Final = _post_in_background(pool, gateway, "/team/delete", {"team_ids": [team_id]}) + eventually( + lambda: _oldest_transaction_blocked_by(holder.info.backend_pid), + lambda age: age > LOCK_HELD_PAST_DEFAULT_TRANSACTION_SECONDS, + seconds=30, + ) + assert not pending.done(), "delete returned while the lock was still held" + return pending.result(timeout=60) + + def _generate_key(scenario: Scenario, **fields: JsonValue) -> str: """A key the team delete is expected to remove; cleanup only deletes it if it is still live.""" key: Final = string_value(scenario.gateway.post("/key/generate", fields)["key"]) @@ -392,6 +438,36 @@ def test_delete_waits_for_the_team_advisory_lock_and_completes_after_release( assert len(_tombstones(team)) == 1, _tombstones(team) +def test_delete_completes_after_waiting_on_a_locked_team_model_row_past_the_default_transaction_timeout( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + team: Final = _create_team(scenario) + model_id: Final = _create_team_model(scenario, team) + response: Final = _delete_while_locked(gateway, team, LOCK_MODEL_ROW_SQL, model_id) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + assert _team_rows(team) == [] + assert _team_model_rows(model_id) == [] + assert len(_tombstones(team)) == 1, _tombstones(team) + + +def test_delete_completes_after_waiting_on_the_team_lock_past_the_default_transaction_timeout( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + user: Final = scenario.user() + team: Final = _create_team(scenario) + _bulk_member_add(gateway, team, [user]) + response: Final = _delete_while_locked(gateway, team, TAKE_TEAM_LOCK_SQL, team) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _user_teams(user) == [] + assert len(_tombstones(team)) == 1, _tombstones(team) + + def test_deleting_two_teams_sharing_a_member_in_one_call_clears_both_from_the_member(gateway: Gateway) -> None: with gateway.scenario() as scenario: user: Final = scenario.user() diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py index cb7f4c2b9ce..e0b2e4a336f 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py @@ -40,6 +40,7 @@ FirstAttempt = Literal[ "drop_after_message_start", "error_frame", "error_frame_after_message_start", + "error_frame_after_content", "http_status", "drop_after_content", ] @@ -66,6 +67,8 @@ class _Failure: return status_reply(self.status) case "drop_after_content": return dropping_reply(chunks, abort_after=3) + case "error_frame_after_content": + return stream_reply((*chunks[:3], error_frame(self.status, f"scripted {self.status}"))) _MID_STREAM_FAILURES: Final = ( @@ -124,12 +127,12 @@ def _stream(gateway: Gateway, body: Mapping[str, JsonValue]) -> tuple[int, tuple return response.status_code, parse_sse(response.text) +def _rows(request_id: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (request_id,)) + + def _success_rows(request_id: str) -> list[dict[str, JsonValue]]: - return eventually( - lambda: read_rows('SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (request_id,)), - lambda rows: len(rows) >= 1, - seconds=70, - ) + return eventually(lambda: _rows(request_id), lambda rows: len(rows) >= 1, seconds=70) def _assert_completed_by_retry( @@ -141,6 +144,8 @@ def _assert_completed_by_retry( assert delta_text(events) == _TEXT, events assert [request.target for request in wire.drain()] == ["/v1/messages"] * attempts assert _success_rows(_served_id(prompt, attempts)) == [{"status": "success", "model_group": model}] + superseded_rows: Final = {attempt: _rows(_served_id(prompt, attempt)) for attempt in range(1, attempts)} + assert not any(superseded_rows.values()), superseded_rows def _assert_rejected_before_content( @@ -302,6 +307,23 @@ def test_stream_dropping_after_content_is_not_retried(gateway: Gateway) -> None: assert [request.target for request in wire.drain()] == ["/v1/messages"] +def test_stream_error_frame_after_content_is_not_retried_and_keeps_its_spend_row(gateway: Gateway) -> None: + prompt: Final = _prompt() + attempts: Final = Attempts() + with ( + wire_server(_upstream(prompt, _Failure("error_frame_after_content", 529), attempts)) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, num_retries=1) + status, events = _stream(gateway, _body(model, prompt, "deployment")) + assert status == 200, events + assert event_types(events) == (*LIFECYCLE[:3], "error"), events + assert message_id(events) == _served_id(prompt, 1), events + assert delta_text(events) == _TEXT, events + assert [request.target for request in wire.drain()] == ["/v1/messages"] + assert [row["model_group"] for row in _success_rows(_served_id(prompt, 1))] == [model] + + def test_stream_rejected_with_401_before_it_opens_is_not_retried(gateway: Gateway) -> None: prompt: Final = _prompt() attempts: Final = Attempts() diff --git a/tests/integration/messages_endpoint/test_anthropic_messages_caller_tools_wire.py b/tests/integration/messages_endpoint/test_anthropic_messages_caller_tools_wire.py index 2e8a9d0d7e4..15c0593f80d 100644 --- a/tests/integration/messages_endpoint/test_anthropic_messages_caller_tools_wire.py +++ b/tests/integration/messages_endpoint/test_anthropic_messages_caller_tools_wire.py @@ -364,7 +364,7 @@ def test_spend_log_keeps_the_tools_the_caller_sent(rig: Rig, model: str, guardra assert object_value(rows[0]["proxy_server_request"])["tools"] == [strict_tool()] -@pytest.mark.parametrize("tools", [7, "x" * 5000, ["lint"]], ids=["int", "five_kb", "list_of_str"]) +@pytest.mark.parametrize("tools", [7, "x" * 5000, ["lint"], ""], ids=["int", "five_kb", "list_of_str", "empty_string"]) def test_a_malformed_tools_field_is_answered_with_an_error_and_the_proxy_keeps_serving( rig: Rig, tools: JsonValue ) -> None: @@ -379,14 +379,6 @@ def test_a_malformed_tools_field_is_answered_with_an_error_and_the_proxy_keeps_s assert reply_text(control_status, control_raw, streaming=False) == f"linted {control_nonce.removeprefix('nonce-')}" -def test_an_empty_string_tools_field_reaches_claude_as_sent(rig: Rig) -> None: - request_nonce: Final = nonce() - status, raw = rig.send(messages_body(CLAUDE_MODEL, request_nonce, tools="")) - - assert reply_text(status, raw, streaming=False) == f"linted {request_nonce.removeprefix('nonce-')}" - assert sent_body(rig.received(request_nonce), CLAUDE_TARGET)["tools"] == "" - - @pytest.mark.parametrize( ("model", "target"), [(CLAUDE_MODEL, CLAUDE_TARGET), (OPENAI_MODEL, RESPONSES_TARGET)], diff --git a/tests/integration/observability/test_generic_guardrail_api_wire.py b/tests/integration/observability/test_generic_guardrail_api_wire.py new file mode 100644 index 00000000000..0796feb5f37 --- /dev/null +++ b/tests/integration/observability/test_generic_guardrail_api_wire.py @@ -0,0 +1,308 @@ +import json +import threading +from collections.abc import Iterator, Mapping +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest +import yaml +from integration._support.anthropic_sse import message_json +from integration._support.client import Gateway, eventually +from integration._support.openai_wire import chat_reply +from integration._support.ports import allocate_port +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +GEMINI_MODEL: Final = "gemini/gemini-3.8-flash" +ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-5-5" +OPENAI_MODEL: Final = "openai/gpt-5.6" +ECHO_GEMINI_TOOLS: Final = "echo-gemini-tools" +NONE_CHAT_ROWS: Final = "none-chat-rows" +NONE_CHAT_ROWS_DEFAULT: Final = "none-chat-rows-default" +NONE_ANTHROPIC_IMAGE: Final = "none-anthropic-image" +ECHO_ANTHROPIC_MESSAGES: Final = "echo-anthropic-messages" +FORWARDS_MESSAGES_AS_SENT: Final = frozenset({NONE_CHAT_ROWS, NONE_ANTHROPIC_IMAGE, ECHO_ANTHROPIC_MESSAGES}) +FAIL_OPEN_CONNECT_ERROR: Final = "fail-open-connect-error" +FAIL_OPEN_TIMEOUT: Final = "fail-open-timeout" +FAIL_OPEN_HTTP_STATUS: Final = "fail-open-http-status" +FAIL_OPEN_LINE: Final = "Generic Guardrail API error (fail-open)" +PNG: Final = ( + "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAFklEQVR4nGO4IydHEmIY1TCqYfhqAACaMxgQdGu1YAAAAABJRU5ErkJggg==" +) +_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" +_JSON: Final = TypeAdapter(dict[str, JsonValue]) + + +def _verdict_none(_request: Request) -> Reply: + return Reply(body=json.dumps({"action": "NONE"}).encode()) + + +def _verdict_echo(request: Request) -> Reply: + body: Final = _JSON.validate_json(request.body) + echoed: Final = {key: body[key] for key in ("structured_messages", "tools") if body.get(key)} + return Reply(body=json.dumps({"action": "NONE", **echoed}).encode()) + + +def _unavailable(_request: Request) -> Reply: + return Reply(status=503, body=b'{"error": "guardrail unavailable"}') + + +def _gemini_reply(_request: Request) -> Reply: + return Reply( + body=json.dumps( + { + "candidates": [{"content": {"role": "model", "parts": [{"text": "ok"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 1, "totalTokenCount": 4}, + } + ).encode() + ) + + +def _anthropic_reply(_request: Request) -> Reply: + return Reply(body=message_json("msg_guardrail_wire", "claude-sonnet-5-5", "ok")) + + +def _openai_reply(_request: Request) -> Reply: + return chat_reply("chatcmpl-guardrail-wire", "gpt-5.6", "ok", stream=False) + + +def _guardrail(name: str, api_base: str, **params: JsonValue) -> dict[str, JsonValue]: + return { + "guardrail_name": name, + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "api_base": api_base, + "api_key": "synthetic-guardrail-key", + **params, + }, + } + + +@dataclass(frozen=True, slots=True) +class GuardrailRig: + owned: OwnedProxy + policies: Mapping[str, Wire] + + +@pytest.fixture(scope="module") +def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[GuardrailRig]: + directory: Final = tmp_path_factory.mktemp("generic-guardrail-wire") + release: Final = threading.Event() + + def held_until_teardown(request: Request) -> Reply: + release.wait() + return _verdict_none(request) + + with ExitStack() as stack: + policies: Final = MappingProxyType( + { + ECHO_GEMINI_TOOLS: stack.enter_context(wire_server(_verdict_echo)), + NONE_CHAT_ROWS: stack.enter_context(wire_server(_verdict_none)), + NONE_CHAT_ROWS_DEFAULT: stack.enter_context(wire_server(_verdict_none)), + NONE_ANTHROPIC_IMAGE: stack.enter_context(wire_server(_verdict_none)), + ECHO_ANTHROPIC_MESSAGES: stack.enter_context(wire_server(_verdict_echo)), + } + ) + stalled: Final = stack.enter_context(wire_server(held_until_teardown)) + stack.callback(release.set) + unavailable: Final = stack.enter_context(wire_server(_unavailable)) + config: Final = _JSON.validate_python(yaml.safe_load(_CONFIG.read_text())) + path: Final = directory / "generic-guardrail-wire.yaml" + path.write_text( + yaml.safe_dump( + { + **config, + "guardrails": [ + *( + _guardrail(name, wire.url, forward_messages_as_sent=name in FORWARDS_MESSAGES_AS_SENT) + for name, wire in policies.items() + ), + _guardrail(FAIL_OPEN_CONNECT_ERROR, f"http://127.0.0.1:{allocate_port()}", fail_on_error=False), + _guardrail(FAIL_OPEN_TIMEOUT, stalled.url, fail_on_error=False, timeout=1), + _guardrail(FAIL_OPEN_HTTP_STATUS, unavailable.url, fail_on_error=False), + ], + } + ) + ) + owned: Final = stack.enter_context(owned_proxy_process(session_gateway, directory, {}, config=path)) + yield GuardrailRig(owned, policies) + + +def _bodies(wire: Wire) -> list[dict[str, JsonValue]]: + return [_JSON.validate_json(request.body) for request in wire.drain()] + + +def _fail_open_lines(log: Path, guardrail: str) -> list[str]: + return [ + line + for line in log.read_text(errors="replace").splitlines() + if FAIL_OPEN_LINE in line and f"guardrail_name={guardrail} " in line + ] + + +def test_gemini_native_tools_reach_the_guardrail_and_gemini_as_sent(rig: GuardrailRig) -> None: + tools: Final = [ + {"function_declarations": [{"name": "get_weather", "parameters": {"type": "object", "properties": {}}}]}, + {"googleSearch": {}}, + ] + with wire_server(_gemini_reply) as gemini, rig.owned.gateway.scenario() as scenario: + model: Final = scenario.model(model=GEMINI_MODEL, api_base=gemini.url, api_key="synthetic-gemini-key") + response: Final = rig.owned.gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": "weather in Paris?"}], + "tools": tools, + "guardrails": [ECHO_GEMINI_TOOLS], + }, + ) + assert response.status_code == 200, response.text + assert [body["tools"] for body in _bodies(rig.policies[ECHO_GEMINI_TOOLS])] == [tools] + sent: Final = _bodies(gemini) + assert len(sent) == 1, sent + assert {"googleSearch": {}} in sent[0]["tools"], sent + assert [tool for tool in sent[0]["tools"] if "type" in tool] == [], sent + + +CHAT_ROWS: Final = ( + {"role": "user", "name": "pat", "content": "weather in Paris?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"index": 0, "id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, +) +OPENAI_CHAT_ROWS: Final = ( + {"role": "user", "content": "weather in Paris?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, +) + + +@pytest.mark.parametrize( + ("guardrail", "seen"), + [(NONE_CHAT_ROWS, CHAT_ROWS), (NONE_CHAT_ROWS_DEFAULT, OPENAI_CHAT_ROWS)], + ids=["forward_messages_as_sent", "default"], +) +def test_guardrail_sees_chat_rows_as_sent_only_when_configured( + rig: GuardrailRig, guardrail: str, seen: tuple[dict[str, JsonValue], ...] +) -> None: + messages: Final = list(CHAT_ROWS) + with wire_server(_openai_reply) as openai_peer, rig.owned.gateway.scenario() as scenario: + model: Final = scenario.model( + model=OPENAI_MODEL, api_base=f"{openai_peer.url}/v1", api_key="synthetic-openai-key" + ) + response: Final = rig.owned.gateway.request( + "POST", "/v1/chat/completions", {"model": model, "messages": messages, "guardrails": [guardrail]} + ) + assert response.status_code == 200, response.text + assert [body["structured_messages"] for body in _bodies(rig.policies[guardrail])] == [list(seen)] + assert len(openai_peer.drain()) == 1 + + +def test_guardrail_sees_an_anthropic_image_block_on_chat_completions_as_sent(rig: GuardrailRig) -> None: + image: Final = [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG}}] + with wire_server(_anthropic_reply) as anthropic_peer, rig.owned.gateway.scenario() as scenario: + model: Final = scenario.model( + model=ANTHROPIC_MODEL, api_base=anthropic_peer.url, api_key="synthetic-anthropic-key" + ) + rig.owned.gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": image}, {"role": "user", "content": "what color?"}], + "guardrails": [NONE_ANTHROPIC_IMAGE], + }, + ) + seen: Final = _bodies(rig.policies[NONE_ANTHROPIC_IMAGE]) + assert [body["structured_messages"][0]["content"] for body in seen] == [image], seen + + +def test_echoing_guardrail_leaves_the_anthropic_messages_request_unchanged(rig: GuardrailRig) -> None: + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What color is this square? Also check the weather in Paris."}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG}}, + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Trip notes."}}, + ], + }, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}}], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": "sunny, 21C"}, + {"type": "text", "text": "Answer both in one sentence."}, + ], + }, + ] + tools: Final = [ + { + "name": "get_weather", + "description": "Weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, + } + ] + with wire_server(_anthropic_reply) as anthropic_peer, rig.owned.gateway.scenario() as scenario: + model: Final = scenario.model( + model=ANTHROPIC_MODEL, api_base=anthropic_peer.url, api_key="synthetic-anthropic-key" + ) + response: Final = rig.owned.gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 64, + "messages": messages, + "tools": tools, + "guardrails": [ECHO_ANTHROPIC_MESSAGES], + }, + ) + assert response.status_code == 200, response.text + assert len(rig.policies[ECHO_ANTHROPIC_MESSAGES].drain()) == 1 + assert [body["messages"] for body in _bodies(anthropic_peer)] == [messages] + + +@pytest.mark.parametrize( + ("guardrail", "tokens"), + [ + (FAIL_OPEN_CONNECT_ERROR, ("error_type=ConnectError",)), + (FAIL_OPEN_TIMEOUT, ("error_type=Timeout",)), + (FAIL_OPEN_HTTP_STATUS, ("error_type=HTTPStatusError", "http_status_code=503")), + ], + ids=["connect_error", "timeout", "http_status"], +) +def test_fail_open_log_line_names_the_error_type(rig: GuardrailRig, guardrail: str, tokens: tuple[str, ...]) -> None: + with wire_server(_openai_reply) as openai_peer, rig.owned.gateway.scenario() as scenario: + model: Final = scenario.model( + model=OPENAI_MODEL, api_base=f"{openai_peer.url}/v1", api_key="synthetic-openai-key" + ) + response: Final = rig.owned.gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "hi"}], "guardrails": [guardrail]}, + ) + assert response.status_code == 200, response.text + assert len(openai_peer.drain()) == 1 + lines: Final = eventually(lambda: _fail_open_lines(rig.owned.log, guardrail), bool, seconds=30) + assert [line for line in lines if not all(token in line for token in tokens)] == [], lines diff --git a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py index 5179514b2b8..701d185b241 100644 --- a/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_count_tokens_wire.py @@ -1147,8 +1147,8 @@ def test_messages_count_tokens_surfaces_the_mantle_error_for_a_non_text_system( @pytest.mark.parametrize( "tools", - ["", "x" * 5120, ["get_weather"]], - ids=["empty_string", "5kb_string", "list_of_strings"], + [5, "", "x" * 5120, ["get_weather"]], + ids=["int", "empty_string", "5kb_string", "list_of_strings"], ) def test_messages_count_tokens_rejects_malformed_tools_without_calling_either_peer( counting_proxy: OwnedProxy, mantle_port: int, tools: JsonValue @@ -1161,8 +1161,8 @@ def test_messages_count_tokens_rejects_malformed_tools_without_calling_either_pe ): model: Final = _deployment(scenario, runtime.url) refused: Final = _count(gateway, {**_bare(model), "tools": tools}) - assert 400 <= refused.status_code < 600, refused.text - assert "input_tokens" not in refused.text, refused.text + assert refused.status_code == 400, refused.text + assert object_value(_JSON_OBJECT.validate_json(refused.content)["error"])["type"] == "invalid_request_error" assert runtime.drain() == () and mantle.drain() == () assert _generated_then_counted(gateway.client, gateway.key, model, _bare(model)) == (200, 200, _MANTLE_COUNT) assert _mantle_bodies(mantle) == (_MANTLE_BARE,) @@ -1185,8 +1185,13 @@ def test_messages_count_tokens_without_messages_is_rejected( assert runtime.drain() == () and mantle.drain() == () +@pytest.mark.parametrize( + "fields", + [{}, {"tools": 5}, {"web_search_options": None}], + ids=["well_formed", "int_tools", "null_web_search_options"], +) def test_messages_count_tokens_unauthenticated_request_never_reaches_either_peer( - counting_proxy: OwnedProxy, mantle_port: int + counting_proxy: OwnedProxy, mantle_port: int, fields: dict[str, JsonValue] ) -> None: gateway: Final = counting_proxy.gateway with ( @@ -1196,7 +1201,7 @@ def test_messages_count_tokens_unauthenticated_request_never_reaches_either_peer ): model: Final = _deployment(scenario, runtime.url) response: Final = gateway.request( - "POST", "/v1/messages/count_tokens", _full(model), key="sk-not-a-key-this-proxy-issued" + "POST", "/v1/messages/count_tokens", {**_full(model), **fields}, key="sk-not-a-key-this-proxy-issued" ) assert response.status_code == 401, response.text assert "input_tokens" not in response.text, response.text diff --git a/tests/integration/providers/test_malformed_tools_chaos.py b/tests/integration/providers/test_malformed_tools_chaos.py new file mode 100644 index 00000000000..0ab50a5afbe --- /dev/null +++ b/tests/integration/providers/test_malformed_tools_chaos.py @@ -0,0 +1,619 @@ +from __future__ import annotations + +import json +import os +import re +import threading +import uuid +from collections.abc import Iterator +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from queue import SimpleQueue +from typing import Final, Literal, TypeAlias + +import httpx +import psutil +import pytest +from integration._support.anthropic_sse import error_body, message_json, message_stream +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.database import write_rows +from integration._support.openai_wire import chat_reply, responses_reply +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from integration._support.worker_readiness import wait_for_worker_readiness +from pydantic import JsonValue, TypeAdapter + +_OPENAI_BACKEND: Final = "gpt-4o-mini" +_CLAUDE_BACKEND: Final = "claude-haiku-4-5" +_GEMINI_BACKEND: Final = "gemini-2.5-flash" +_PROVIDER_KEY: Final = "scripted-malformed-tools-chaos-key" +_OWNED_CHAT: Final = "malformed-tools-owned-chat" +_OWNED_STORE: Final = "vs_malformed_tools_owned_store" +_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})") +_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") +_OWNED_PROXY_CELL_SECONDS: Final = 2 * graceful_stop_seconds() + 120 +_RESPAWN_SECONDS: Final = graceful_stop_seconds() + 60 +_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1"))) +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_NOT_A_LIST: Final = "tools: Input should be a valid list" +_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], +} +_CHAT_TOOL: Final[dict[str, JsonValue]] = { + "type": "function", + "function": {"name": "get_weather", "description": "Look up the weather", "parameters": _WEATHER_SCHEMA}, +} +_CLAUDE_TOOL: Final[dict[str, JsonValue]] = { + "name": "get_weather", + "description": "Look up the weather", + "input_schema": _WEATHER_SCHEMA, +} +_RESPONSES_TOOL: Final[dict[str, JsonValue]] = { + "type": "function", + "name": "get_weather", + "description": "Look up the weather", + "parameters": _WEATHER_SCHEMA, +} +_GEMINI_TOOL: Final[dict[str, JsonValue]] = { + "functionDeclarations": [ + {"name": "get_weather", "description": "Look up the weather", "parameters": _WEATHER_SCHEMA} + ] +} +Kind: TypeAlias = Literal["valid", "rejected", "dropped"] + + +def _marker(label: str) -> str: + return f"{label} marker-{uuid.uuid4().hex}" + + +def _tag(text: str) -> str: + found: Final = _MARKER.search(text) + return found.group(1) if found else "none" + + +def _answer(tag: str) -> str: + return f"answer {tag}" + + +def _gemini_body(tag: str) -> bytes: + return json.dumps( + { + "candidates": [ + {"content": {"role": "model", "parts": [{"text": _answer(tag)}]}, "index": 0, "finishReason": "STOP"} + ], + "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 2, "totalTokenCount": 11}, + "modelVersion": _GEMINI_BACKEND, + } + ).encode() + + +def _scripted(request: Request) -> Reply: + text: Final = request.body.decode() + tag: Final = _tag(text) + path: Final = request.target.split("?")[0] + stream: Final = bool(text) and _JSON_OBJECT.validate_json(request.body).get("stream") is True + if path == "/v1/chat/completions": + return chat_reply(f"chatcmpl-{tag}", _OPENAI_BACKEND, _answer(tag), stream=stream) + if path == "/v1/responses": + return responses_reply(f"resp_{tag}", _OPENAI_BACKEND, _answer(tag), stream=stream) + if path == "/v1/messages" and _JSON_OBJECT.validate_json(request.body).get("tools") == 5: + return Reply(status=400, body=error_body(400, _NOT_A_LIST)) + if path == "/v1/messages" and stream: + return Reply( + content_type="text/event-stream", chunks=message_stream(f"msg_{tag}", _CLAUDE_BACKEND, _answer(tag)) + ) + if path == "/v1/messages": + return Reply(body=message_json(f"msg_{tag}", _CLAUDE_BACKEND, _answer(tag))) + if path.endswith(":streamGenerateContent"): + return Reply(content_type="text/event-stream", chunks=(b"data: " + _gemini_body(tag) + b"\r\n\r\n",)) + if path.endswith(":generateContent"): + return Reply(body=_gemini_body(tag)) + return Reply(status=404, body=json.dumps({"error": f"unscripted {request.target}"}).encode()) + + +@dataclass(frozen=True, slots=True) +class _Switchboard: + down: threading.Event + holding: threading.Event + release: threading.Event + held: SimpleQueue[str] + + def respond(self, request: Request) -> Reply: + if self.down.is_set(): + return Reply(drop_connection=True) + if self.holding.is_set(): + self.held.put(request.target) + assert self.release.wait(timeout=120), "Held request was never released" + return _scripted(request) + + def reset(self) -> None: + self.down.clear() + self.release.set() + self.holding.clear() + + +@dataclass(frozen=True, slots=True) +class _Rig: + gateway: Gateway + wire: Wire + switchboard: _Switchboard + chat: str + claude: str + gemini: str + + +@pytest.fixture(scope="module") +def rig() -> Iterator[_Rig]: + switchboard: Final = _Switchboard(threading.Event(), threading.Event(), threading.Event(), SimpleQueue()) + with ( + gateway_from_environment() as gateway, + wire_server(switchboard.respond) as wire, + gateway.scenario() as scenario, + ): + built: Final = _Rig( + gateway=gateway, + wire=wire, + switchboard=switchboard, + chat=scenario.model(api_base=f"{wire.url}/v1", api_key=_PROVIDER_KEY), + claude=scenario.model(model=f"anthropic/{_CLAUDE_BACKEND}", api_base=wire.url, api_key=_PROVIDER_KEY), + gemini=scenario.model(model=f"gemini/{_GEMINI_BACKEND}", api_base=wire.url, api_key=_PROVIDER_KEY), + ) + for group in (built.chat, built.claude, built.gemini): + wait_for_worker_readiness(gateway, group, 1, _PROXY_WORKERS) + try: + yield built + finally: + switchboard.reset() + + +@pytest.fixture(autouse=True) +def _reset(rig: _Rig) -> Iterator[None]: + rig.switchboard.reset() + rig.wire.drain() + yield + rig.switchboard.reset() + + +@dataclass(frozen=True, slots=True) +class _Call: + path: str + body: dict[str, JsonValue] + marker: str + kind: Kind + + +def _tools_for(kind: Kind, tool: dict[str, JsonValue]) -> dict[str, JsonValue]: + if kind == "rejected": + return {"tools": 5} + if kind == "dropped": + return {"tools": [tool, 5], "drop_params": True} + return {"tools": [tool]} + + +def _call(rig: _Rig, index: int, kind: Kind, label: str) -> _Call: + marker: Final = _marker(label) + stream: Final = (index // 4) % 2 == 1 + family: Final = index % 4 + if family == 0: + return _Call( + "/v1/chat/completions", + { + "model": rig.chat, + "messages": [{"role": "user", "content": marker}], + "stream": stream, + **_tools_for(kind, _CHAT_TOOL), + }, + marker, + kind, + ) + if family == 1: + return _Call( + "/v1/messages", + { + "model": rig.claude, + "max_tokens": 16, + "messages": [{"role": "user", "content": marker}], + "stream": stream, + **_tools_for(kind, _CLAUDE_TOOL), + }, + marker, + kind, + ) + if family == 2: + return _Call( + "/v1/responses", + {"model": rig.chat, "input": marker, "stream": stream, **_tools_for(kind, _RESPONSES_TOOL)}, + marker, + kind, + ) + action: Final = "streamGenerateContent?alt=sse" if stream else "generateContent" + return _Call( + f"/v1beta/models/{rig.gemini}:{action}", + {"contents": [{"role": "user", "parts": [{"text": marker}]}], **_tools_for(kind, _GEMINI_TOOL)}, + marker, + kind, + ) + + +def _kind(index: int) -> Kind: + kinds: Final[tuple[Kind, ...]] = ("valid", "rejected", "dropped") + return kinds[index % 3] + + +def _clients(stack: ExitStack, base_url: str, count: int) -> tuple[httpx.Client, ...]: + return tuple( + stack.enter_context(httpx.Client(base_url=base_url, timeout=120, trust_env=False)) for _ in range(count) + ) + + +def _post(client: httpx.Client, key: str, call: _Call) -> httpx.Response: + return client.post(call.path, json=call.body, headers={"Authorization": f"Bearer {key}"}) + + +def _status_or_dropped(client: httpx.Client, key: str, call: _Call) -> int | None: + try: + return _post(client, key, call).status_code + except httpx.TransportError: + return None + + +def _deliveries(wire: Wire) -> tuple[str, ...]: + return tuple(request.body.decode() for request in wire.drain()) + + +def _seen(deliveries: tuple[str, ...], marker: str) -> int: + return sum(marker in body for body in deliveries) + + +def _delivered_tool_names(deliveries: tuple[str, ...], marker: str) -> tuple[str, ...]: + (body,) = tuple(body for body in deliveries if marker in body) + return tuple(re.findall(r'"name": ?"([a-z_]+)"', body)) + + +def _assert_wave(calls: tuple[_Call, ...], statuses: tuple[int | None, ...], deliveries: tuple[str, ...]) -> None: + expected_statuses: Final = tuple(400 if call.kind == "rejected" else 200 for call in calls) + assert statuses == expected_statuses, tuple(zip(statuses, (call.marker for call in calls), strict=True)) + seen: Final = tuple(_seen(deliveries, call.marker) for call in calls) + assert seen == tuple(0 if call.kind == "rejected" else 1 for call in calls), seen + names: Final = tuple(_delivered_tool_names(deliveries, call.marker) for call in calls if call.kind != "rejected") + assert all("get_weather" in found for found in names), names + + +def test_mixed_burst_rejects_malformed_tools_and_forwards_the_rest_once(rig: _Rig) -> None: + with ExitStack() as stack: + calls: Final = tuple(_call(rig, index, _kind(index), "burst") for index in range(40)) + clients: Final = _clients(stack, str(rig.gateway.client.base_url), len(calls)) + pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(calls))) + statuses: Final = tuple( + pool.map( + lambda pair: _status_or_dropped(pair[0], rig.gateway.key, pair[1]), zip(clients, calls, strict=True) + ) + ) + _assert_wave(calls, statuses, _deliveries(rig.wire)) + + +def test_upstream_outage_still_rejects_malformed_tools_and_recovers(rig: _Rig) -> None: + with ExitStack() as stack: + clients: Final = _clients(stack, str(rig.gateway.client.base_url), 12) + pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients))) + + def wave(label: str) -> tuple[tuple[_Call, ...], tuple[int | None, ...]]: + calls: Final = tuple(_call(rig, index, _kind(index), label) for index in range(len(clients))) + statuses: Final = tuple( + pool.map( + lambda pair: _status_or_dropped(pair[0], rig.gateway.key, pair[1]), zip(clients, calls, strict=True) + ) + ) + return calls, statuses + + before_calls, before_statuses = wave("before outage") + _assert_wave(before_calls, before_statuses, _deliveries(rig.wire)) + rig.switchboard.down.set() + outage_calls, outage_statuses = wave("during outage") + for call, status in zip(outage_calls, outage_statuses, strict=True): + assert (status == 400) if call.kind == "rejected" else (status is not None and status >= 500), ( + call.kind, + status, + ) + assert rig.gateway.request("GET", "/health/liveliness").status_code == 200 + rig.wire.drain() + rig.switchboard.down.clear() + after_calls, after_statuses = wave("after outage") + _assert_wave(after_calls, after_statuses, _deliveries(rig.wire)) + + +def test_held_upstream_does_not_delay_malformed_tools_rejections(rig: _Rig) -> None: + with ExitStack() as stack: + held_calls: Final = tuple(_call(rig, index * 4, "valid", "held") for index in range(8)) + rejected_calls: Final = tuple(_call(rig, index, "rejected", "rejected") for index in range(8)) + clients: Final = _clients(stack, str(rig.gateway.client.base_url), len(held_calls) + len(rejected_calls)) + pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients))) + rig.switchboard.release.clear() + rig.switchboard.holding.set() + stack.callback(rig.switchboard.release.set) + pending: Final[tuple[Future[int | None], ...]] = tuple( + pool.submit(_status_or_dropped, client, rig.gateway.key, call) + for client, call in zip(clients[: len(held_calls)], held_calls, strict=True) + ) + eventually(rig.switchboard.held.qsize, lambda size: size == len(held_calls), seconds=60) + rejected: Final = tuple( + pool.map( + lambda pair: _status_or_dropped(pair[0], rig.gateway.key, pair[1]), + zip(clients[len(held_calls) :], rejected_calls, strict=True), + ) + ) + assert rejected == (400,) * len(rejected_calls), rejected + assert not any(future.done() for future in pending), "a held request finished before its release" + rig.switchboard.release.set() + assert tuple(future.result(timeout=120) for future in pending) == (200,) * len(held_calls) + deliveries: Final = _deliveries(rig.wire) + assert tuple(_seen(deliveries, call.marker) for call in held_calls) == (1,) * len(held_calls) + assert tuple(_seen(deliveries, call.marker) for call in rejected_calls) == (0,) * len(rejected_calls) + + +@dataclass(frozen=True, slots=True) +class _Owned: + proxy: OwnedProxy + wire: Wire + held: SimpleQueue[str] + holding: threading.Event + release: threading.Event + blocked_tool: str + denied_tool: str + + +def _owned_config(path: Path, upstream: str, denied_tool: str) -> Path: + path.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": _OWNED_CHAT, + "litellm_params": { + "model": f"openai/{_OPENAI_BACKEND}", + "api_base": f"{upstream}/v1", + "api_key": _PROVIDER_KEY, + }, + } + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + }, + "litellm_settings": {"drop_params": True}, + "router_settings": {"disable_cooldowns": True}, + "guardrails": [ + { + "guardrail_name": "malformed-tools-policy", + "litellm_params": {"guardrail": "tool_policy", "mode": "pre_call", "default_on": True}, + }, + { + "guardrail_name": "malformed-tools-permission", + "litellm_params": { + "guardrail": "tool_permission", + "mode": "pre_call", + "default_on": False, + "default_action": "allow", + "on_disallowed_action": "block", + "rules": [{"id": "deny_tool", "tool_name": f"^{denied_tool}$", "decision": "deny"}], + }, + }, + ], + "vector_store_registry": [ + { + "vector_store_name": "malformed-tools-owned-store", + "litellm_params": { + "vector_store_id": _OWNED_STORE, + "custom_llm_provider": "openai", + "api_base": upstream, + "api_key": _PROVIDER_KEY, + }, + } + ], + } + ) + ) + return path + + +@pytest.fixture(scope="module") +def owned(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Owned]: + suffix: Final = uuid.uuid4().hex[:12] + blocked_tool: Final = f"wipe_disk_{suffix}" + denied_tool: Final = f"run_shell_{suffix}" + held: Final[SimpleQueue[str]] = SimpleQueue() + holding: Final = threading.Event() + release: Final = threading.Event() + + def respond(request: Request) -> Reply: + if holding.is_set(): + held.put(request.target) + assert release.wait(timeout=120), "Held request was never released" + return _scripted(request) + + directory: Final = tmp_path_factory.mktemp("malformed-tools-owned") + with ExitStack() as stack: + gateway: Final = stack.enter_context(gateway_from_environment()) + policy: Final = gateway.request( + "POST", "/v1/tool/policy", {"tool_name": blocked_tool, "input_policy": "blocked"} + ) + assert policy.status_code == 200, policy.text + stack.callback(write_rows, 'DELETE FROM "LiteLLM_ToolTable" WHERE tool_name = %s', (blocked_tool,)) + wire: Final = stack.enter_context(wire_server(respond)) + stack.callback(release.set) + proxy: Final = stack.enter_context( + owned_proxy_process( + gateway, + directory, + {"ANTHROPIC_API_BASE": wire.url, "ANTHROPIC_API_KEY": _PROVIDER_KEY}, + config=_owned_config(directory / "malformed-tools.yaml", wire.url, denied_tool), + workers=2, + ) + ) + yield _Owned(proxy, wire, held, holding, release, blocked_tool, denied_tool) + + +def _owned_chat(owned: _Owned, marker: str, **fields: JsonValue) -> httpx.Response: + return owned.proxy.gateway.request( + "POST", + "/v1/chat/completions", + {"model": _OWNED_CHAT, "messages": [{"role": "user", "content": marker}], **fields}, + ) + + +def _named_tool(name: str) -> dict[str, JsonValue]: + return {"type": "function", "function": {"name": name, "parameters": _WEATHER_SCHEMA}} + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_global_drop_params_wins_over_an_explicit_request_false(owned: _Owned) -> None: + marker: Final = _marker("global drop") + response: Final = _owned_chat(owned, marker, tools=5, drop_params=False) + assert response.status_code == 200, response.text + assert _answer(_tag(marker)) in response.text, response.text + (sent,) = tuple(request for request in owned.wire.drain() if marker in request.body.decode()) + assert "tools" not in _JSON_OBJECT.validate_json(sent.body), sent.body + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_tool_policy_still_blocks_a_tool_kept_beside_a_dropped_entry(owned: _Owned) -> None: + marker: Final = _marker("tool policy") + response: Final = _owned_chat(owned, marker, tools=[_named_tool(owned.blocked_tool), 5]) + assert response.status_code == 400, response.text + assert "Violated tool policy" in response.text and owned.blocked_tool in response.text, response.text + assert tuple(request for request in owned.wire.drain() if marker in request.body.decode()) == () + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_tool_permission_still_denies_a_tool_kept_beside_a_dropped_entry(owned: _Owned) -> None: + marker: Final = _marker("tool permission") + response: Final = _owned_chat( + owned, marker, tools=[_named_tool(owned.denied_tool), "x"], guardrails=["malformed-tools-permission"] + ) + assert response.status_code == 400, response.text + assert "Violated guardrail policy" in response.text and owned.denied_tool in response.text, response.text + assert tuple(request for request in owned.wire.drain() if marker in request.body.decode()) == () + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_anthropic_pass_through_forwards_malformed_tools_unchanged(owned: _Owned) -> None: + marker: Final = _marker("pass-through") + body: Final[dict[str, JsonValue]] = { + "model": _CLAUDE_BACKEND, + "max_tokens": 16, + "messages": [{"role": "user", "content": marker}], + "tools": 5, + } + response: Final = owned.proxy.gateway.request("POST", "/anthropic/v1/messages", body) + assert response.status_code == 400, response.text + assert _JSON_OBJECT.validate_json(response.content) == _JSON_OBJECT.validate_json(error_body(400, _NOT_A_LIST)) + (sent,) = tuple(request for request in owned.wire.drain() if marker in request.body.decode()) + assert sent.target == "/v1/messages", sent.target + assert _JSON_OBJECT.validate_json(sent.body) == body, sent.body + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_anthropic_pass_through_streams_well_formed_tools(owned: _Owned) -> None: + marker: Final = _marker("pass-through stream") + body: Final[dict[str, JsonValue]] = { + "model": _CLAUDE_BACKEND, + "max_tokens": 16, + "messages": [{"role": "user", "content": marker}], + "tools": [_CLAUDE_TOOL], + "stream": True, + } + response: Final = owned.proxy.gateway.request("POST", "/anthropic/v1/messages", body) + assert response.status_code == 200, response.text + assert response.content == b"".join(message_stream(f"msg_{_tag(marker)}", _CLAUDE_BACKEND, _answer(_tag(marker)))) + (sent,) = tuple(request for request in owned.wire.drain() if marker in request.body.decode()) + assert _JSON_OBJECT.validate_json(sent.body) == body, sent.body + + +def _local_port(client: httpx.Client) -> int: + with client.stream("GET", "/health/liveliness") as response: + port: Final = int(response.extensions["network_stream"].get_extra_info("client_addr")[1]) + response.read() + assert response.status_code == 200, response.text + return port + + +def _accepted_client_ports(pid: int, proxy_port: int) -> frozenset[int]: + return frozenset( + connection.raddr.port + for connection in psutil.Process(pid).net_connections(kind="tcp") + if connection.raddr and connection.laddr.port == proxy_port + ) + + +def _owned_call(index: int, label: str) -> _Call: + marker: Final = _marker(label) + tools: Final[JsonValue] = [_CHAT_TOOL, 5] if index % 2 else [_CHAT_TOOL] + return _Call( + "/v1/chat/completions", + { + "model": _OWNED_CHAT, + "messages": [{"role": "user", "content": marker}], + "stream": index % 4 == 1, + "tools": tools, + }, + marker, + "dropped" if index % 2 else "valid", + ) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_worker_sigkill_mid_burst_leaves_the_replacement_dropping_bad_entries(owned: _Owned) -> None: + proxy_url: Final = owned.proxy.gateway.client.base_url + workers: Final = eventually( + lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.proxy.log.read_text())), + lambda pids: len(pids) >= 2, + seconds=30, + ) + started_before: Final = len(workers) + serving: Final = owned.proxy.worker_pids() + owned.wire.drain() + with ExitStack() as stack: + clients: Final = _clients(stack, str(proxy_url), 12) + pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients))) + ports: Final = tuple(_local_port(client) for client in clients) + calls: Final = tuple(_owned_call(index, "worker kill") for index in range(len(clients))) + owned.release.clear() + owned.holding.set() + stack.callback(owned.holding.clear) + stack.callback(owned.release.set) + futures: Final = tuple( + pool.submit(_status_or_dropped, client, owned.proxy.gateway.key, call) + for client, call in zip(clients, calls, strict=True) + ) + eventually(owned.held.qsize, lambda size: size == len(clients), seconds=60) + shares: Final = {pid: _accepted_client_ports(pid, proxy_url.port or 0) & frozenset(ports) for pid in serving} + assert sum(map(len, shares.values())) == len(clients), shares + victim: Final = min((pid for pid in serving if shares[pid]), key=lambda pid: len(shares[pid])) + with owned.proxy.replacing_worker(victim) as kill: + kill() + owned.holding.clear() + owned.release.set() + results: Final = tuple(future.result(timeout=120) for future in futures) + for port, result in zip(ports, results, strict=True): + assert result == (None if port in shares[victim] else 200), (port, result, shares) + eventually( + lambda: len(_STARTED_WORKER.findall(owned.proxy.log.read_text())), + lambda started: started > started_before, + _RESPAWN_SECONDS, + ) + second_wave: Final = tuple(_owned_call(index * 2 + 1, "after kill") for index in range(6)) + (fresh,) = _clients(stack, str(proxy_url), 1) + assert tuple(_post(fresh, owned.proxy.gateway.key, call).status_code for call in second_wave) == (200,) * len( + second_wave + ) + deliveries: Final = _deliveries(owned.wire) + everything: Final = calls + second_wave + assert tuple(_seen(deliveries, call.marker) for call in everything) == (1,) * len(everything) + assert tuple(_delivered_tool_names(deliveries, call.marker) for call in everything) == ( + ("get_weather",), + ) * len(everything) + assert owned.proxy.process.poll() is None diff --git a/tests/integration/providers/test_malformed_tools_wire.py b/tests/integration/providers/test_malformed_tools_wire.py new file mode 100644 index 00000000000..25a86f7fc7c --- /dev/null +++ b/tests/integration/providers/test_malformed_tools_wire.py @@ -0,0 +1,2731 @@ +from __future__ import annotations + +import asyncio +import http.client +import json +import os +import re +import uuid +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, replace +from itertools import chain, product +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import anthropic +import httpx +import openai +import pytest +from google import genai +from google.genai import types +from integration._support.anthropic_sse import error_body, message_json, message_stream +from integration._support.client import ( + Gateway, + Scenario, + eventually, + gateway_from_environment, + held, + object_value, + string_value, +) +from integration._support.database import read_rows +from integration._support.openai_wire import chat_reply, openai_error, responses_reply +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.responses_vendor import same_response +from integration._support.wire import Reply, Request, Wire, wire_server +from integration._support.worker_readiness import wait_for_worker_readiness +from integration.authorization._guardrail_opt_out import upstream_observations +from pydantic import JsonValue, TypeAdapter + +from tests.integration.caching.response_cache_case import await_entry, redis_store + +_OPENAI_BACKEND: Final = "gpt-4o-mini" +_CLAUDE_BACKEND: Final = "claude-haiku-4-5" +_GEMINI_BACKEND: Final = "gemini-2.5-flash" +_PROVIDER_KEY: Final = "scripted-malformed-tools-key" +_COUNT: Final = 37 +_CONFIG_STORE: Final = "vs_integration_config_store" +_OTHER_STORE: Final = "vs_malformed_tools_other_store" +_CLAUDE_CODE_AGENT: Final = "claude-cli/2.1.0 (external, cli)" +_CODEX_AGENT: Final = "codex_cli_rs/0.50.0 (Linux 6.8; x86_64) xterm-256color" +_REJECT_MARK: Final = "upstream-401" +_DOWN: Final = "/down/" +_REFUSALS: Final = MappingProxyType( + { + "context_window_fallbacks": ( + "/context-full/", + { + "message": "This model's maximum context length is 8192 tokens.", + "type": "invalid_request_error", + "param": "messages", + "code": "context_length_exceeded", + }, + ), + "content_policy_fallbacks": ( + "/content-policy/", + { + "message": "Your request was rejected as a result of our safety system.", + "type": "invalid_request_error", + "param": None, + "code": "content_policy_violation", + }, + ), + } +) +_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})") +_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1"))) +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_SPEND_ROWS: Final = ( + 'SELECT call_type, status, cache_hit FROM "LiteLLM_SpendLogs" WHERE request_id = %s OR litellm_call_id = %s' +) +_NOT_A_LIST: Final = "tools: Input should be a valid list" +_NOT_AN_OBJECT: Final = "tools.0: Input should be a valid dictionary" +_SECOND_NOT_AN_OBJECT: Final = "tools.1: Input should be a valid dictionary" +_OWNED_PROXY_CELL_SECONDS: Final = 2 * graceful_stop_seconds() + 120 +_TOOL_FIELDS: Final = ("tools", "tool_choice", "parallel_tool_calls", "toolConfig", "tool_config") +_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], +} + + +def _marker() -> str: + return f"marker-{uuid.uuid4().hex}" + + +def _tag(text: str) -> str: + found: Final = _MARKER.search(text) + return found.group(1) if found else "none" + + +def _answer(tag: str) -> str: + return f"answer {tag}" + + +def _gemini_body(tag: str) -> dict[str, JsonValue]: + return { + "responseId": f"gemini-{tag}-{uuid.uuid4().hex}", + "candidates": [ + {"content": {"role": "model", "parts": [{"text": _answer(tag)}]}, "index": 0, "finishReason": "STOP"} + ], + "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 2, "totalTokenCount": 11}, + "modelVersion": _GEMINI_BACKEND, + } + + +def _interaction_body(tag: str) -> dict[str, JsonValue]: + return { + "id": f"interaction-{tag}", + "object": "interaction", + "model": _GEMINI_BACKEND, + "status": "completed", + "created": "2026-10-15T00:00:00Z", + "steps": [{"type": "content", "content": {"type": "text", "text": _answer(tag)}}], + "usage": {"total_tokens": 11}, + } + + +def _rejected_by_provider(path: str) -> Reply: + if path.startswith("/v1/messages"): + return Reply(status=401, body=error_body(401, "scripted 401")) + if path == "/v1beta/interactions": + return Reply( + status=401, + body=json.dumps({"error": {"code": 401, "message": "scripted 401", "status": "UNAUTHENTICATED"}}).encode(), + ) + return openai_error(401) + + +def _unavailable(path: str) -> Reply: + if "/v1/messages" in path: + return Reply(status=503, body=error_body(503, "scripted outage")) + return openai_error(503) + + +def _respond(request: Request) -> Reply: + text: Final = request.body.decode() + tag: Final = _tag(text) + path: Final = request.target.split("?")[0] + if path.startswith(_DOWN): + return _unavailable(path) + refusal: Final = next((error for prefix, error in _REFUSALS.values() if path.startswith(prefix)), None) + if refusal is not None: + return Reply(status=400, body=json.dumps({"error": refusal}).encode()) + if _REJECT_MARK in text: + return _rejected_by_provider(path) + stream: Final = bool(text) and _JSON_OBJECT.validate_json(request.body).get("stream") is True + if path == "/v1/chat/completions": + return chat_reply(f"chatcmpl-{tag}", _OPENAI_BACKEND, _answer(tag), stream=stream) + if path == "/v1/responses": + return responses_reply(f"resp_{tag}", _OPENAI_BACKEND, _answer(tag), stream=stream) + if path == "/v1/messages/count_tokens": + return Reply(body=json.dumps({"input_tokens": _COUNT}).encode()) + if path == "/v1/messages" and stream: + return Reply( + content_type="text/event-stream", chunks=message_stream(f"msg_{tag}", _CLAUDE_BACKEND, _answer(tag)) + ) + if path == "/v1/messages": + return Reply(body=message_json(f"msg_{tag}", _CLAUDE_BACKEND, _answer(tag))) + if path.endswith(":streamGenerateContent"): + return Reply( + content_type="text/event-stream", chunks=(f"data: {json.dumps(_gemini_body(tag))}\r\n\r\n".encode(),) + ) + if path.endswith(":generateContent"): + return Reply(body=json.dumps(_gemini_body(tag)).encode()) + if path == "/v1beta/interactions": + return Reply(body=json.dumps(_interaction_body(tag)).encode()) + return Reply(status=404, body=json.dumps({"error": f"unscripted {request.target}"}).encode()) + + +@dataclass(frozen=True, slots=True) +class _Rig: + gateway: Gateway + wire: Wire + openai: str + anthropic: str + gemini: str + drop_group: str + gemini_drop_group: str + mixed_group: str + field_group: str + tools_drop_group: str + + +def _deployment_ids(scenario: Scenario, name: str, deployments: tuple[dict[str, JsonValue], ...]) -> tuple[str, ...]: + created: Final = tuple( + scenario.gateway.post("/model/new", {"model_name": name, "litellm_params": params, "model_info": {}}) + for params in deployments + ) + identities: Final = tuple(string_value(object_value(deployment["model_info"])["id"]) for deployment in created) + for identity in identities: + scenario.cleanups.callback(scenario.delete_model, identity) + return identities + + +def _same_name_deployments(scenario: Scenario, api_base: str, variants: tuple[Mapping[str, JsonValue], ...]) -> str: + name: Final = f"integration-mixed-{uuid.uuid4().hex}" + _deployment_ids( + scenario, + name, + tuple( + {"model": f"openai/{_OPENAI_BACKEND}", "api_key": _PROVIDER_KEY, "api_base": api_base, **variant} + for variant in variants + ), + ) + return name + + +@pytest.fixture(scope="module") +def rig() -> Iterator[_Rig]: + with gateway_from_environment() as gateway, wire_server(_respond) as wire, gateway.scenario() as scenario: + openai_base: Final = f"{wire.url}/v1" + built: Final = _Rig( + gateway=gateway, + wire=wire, + openai=scenario.model(api_base=openai_base, api_key=_PROVIDER_KEY), + anthropic=scenario.model(model=f"anthropic/{_CLAUDE_BACKEND}", api_base=wire.url, api_key=_PROVIDER_KEY), + gemini=scenario.model(model=f"gemini/{_GEMINI_BACKEND}", api_base=wire.url, api_key=_PROVIDER_KEY), + drop_group=scenario.model(api_base=openai_base, api_key=_PROVIDER_KEY, drop_params=True), + gemini_drop_group=scenario.model( + model=f"gemini/{_GEMINI_BACKEND}", api_base=wire.url, api_key=_PROVIDER_KEY, drop_params=True + ), + mixed_group=_same_name_deployments(scenario, openai_base, ({"drop_params": True}, {})), + field_group=scenario.model(api_base=openai_base, api_key=_PROVIDER_KEY, additional_drop_params=["tools"]), + tools_drop_group=scenario.model( + api_base=openai_base, api_key=_PROVIDER_KEY, drop_params=True, tools=[CHAT.tool] + ), + ) + for group, deployments in ( + (built.openai, 1), + (built.anthropic, 1), + (built.gemini, 1), + (built.drop_group, 1), + (built.gemini_drop_group, 1), + (built.mixed_group, 2), + (built.field_group, 1), + (built.tools_drop_group, 1), + ): + wait_for_worker_readiness(gateway, group, deployments, _PROXY_WORKERS) + yield built + + +@pytest.fixture(autouse=True) +def _drained(rig: _Rig) -> None: + rig.wire.drain() + + +@dataclass(frozen=True, slots=True) +class _FieldGroups: + anthropic: str + gemini: str + + +@pytest.fixture(scope="module") +def field_groups(rig: _Rig) -> Iterator[_FieldGroups]: + with rig.gateway.scenario() as scenario: + built: Final = _FieldGroups( + anthropic=scenario.model( + model=f"anthropic/{_CLAUDE_BACKEND}", + api_base=rig.wire.url, + api_key=_PROVIDER_KEY, + additional_drop_params=["tools"], + ), + gemini=scenario.model( + model=f"gemini/{_GEMINI_BACKEND}", + api_base=rig.wire.url, + api_key=_PROVIDER_KEY, + additional_drop_params=["tools"], + ), + ) + for group in (built.anthropic, built.gemini): + wait_for_worker_readiness(rig.gateway, group, 1, _PROXY_WORKERS) + yield built + + +@pytest.fixture(scope="module") +def limited_key(rig: _Rig) -> Iterator[str]: + with rig.gateway.scenario() as scenario: + yield scenario.key(rpm_limit=0) + + +@dataclass(frozen=True, slots=True) +class _Family: + name: str + tool: dict[str, JsonValue] + model: Callable[[_Rig], str] + path: Callable[[str, bool], str] + body: Callable[[str, str, bool], dict[str, JsonValue]] + target: Callable[[bool], str] + call_type: Callable[[bool], str] + upstream_id: Callable[[str], str | None] + tool_choice: tuple[str, JsonValue] + exact_tools: bool + + +def _chat_body(model: str, marker: str, stream: bool) -> dict[str, JsonValue]: + return {"model": model, "messages": [{"role": "user", "content": marker}], "stream": stream} + + +def _messages_body(model: str, marker: str, stream: bool) -> dict[str, JsonValue]: + return {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": marker}], "stream": stream} + + +def _responses_body(model: str, marker: str, stream: bool) -> dict[str, JsonValue]: + return {"model": model, "input": marker, "stream": stream} + + +def _generate_body(model: str, marker: str, stream: bool) -> dict[str, JsonValue]: + return {"contents": [{"role": "user", "parts": [{"text": marker}]}]} + + +def _generate_path(model: str, stream: bool) -> str: + return ( + f"/v1beta/models/{model}:streamGenerateContent?alt=sse" if stream else f"/v1beta/models/{model}:generateContent" + ) + + +CHAT: Final = _Family( + name="chat", + tool={ + "type": "function", + "function": {"name": "get_weather", "description": "Look up the weather", "parameters": _WEATHER_SCHEMA}, + }, + model=lambda rig: rig.openai, + path=lambda model, stream: "/v1/chat/completions", + body=_chat_body, + target=lambda stream: "/v1/chat/completions", + call_type=lambda stream: "acompletion", + upstream_id=lambda tag: f"chatcmpl-{tag}", + tool_choice=("tool_choice", {"type": "function", "function": {"name": "get_weather"}}), + exact_tools=True, +) +MESSAGES: Final = _Family( + name="messages", + tool={"name": "get_weather", "description": "Look up the weather", "input_schema": _WEATHER_SCHEMA}, + model=lambda rig: rig.anthropic, + path=lambda model, stream: "/v1/messages", + body=_messages_body, + target=lambda stream: "/v1/messages", + call_type=lambda stream: "anthropic_messages", + upstream_id=lambda tag: f"msg_{tag}", + tool_choice=("tool_choice", {"type": "tool", "name": "get_weather"}), + exact_tools=True, +) +RESPONSES: Final = _Family( + name="responses", + tool={ + "type": "function", + "name": "get_weather", + "description": "Look up the weather", + "parameters": _WEATHER_SCHEMA, + }, + model=lambda rig: rig.openai, + path=lambda model, stream: "/v1/responses", + body=_responses_body, + target=lambda stream: "/v1/responses", + call_type=lambda stream: "aresponses", + upstream_id=lambda tag: None, + tool_choice=("tool_choice", {"type": "function", "name": "get_weather"}), + exact_tools=True, +) +GENERATE: Final = _Family( + name="generate_content", + tool={ + "functionDeclarations": [ + {"name": "get_weather", "description": "Look up the weather", "parameters": _WEATHER_SCHEMA} + ] + }, + model=lambda rig: rig.gemini, + path=_generate_path, + body=_generate_body, + target=lambda stream: ":streamGenerateContent" if stream else ":generateContent", + call_type=lambda stream: "agenerate_content_stream" if stream else "agenerate_content", + upstream_id=lambda tag: None, + tool_choice=("toolConfig", {"functionCallingConfig": {"mode": "ANY", "allowedFunctionNames": ["get_weather"]}}), + exact_tools=False, +) +_FAMILIES: Final = (CHAT, MESSAGES, RESPONSES, GENERATE) +_FAMILY_IDS: Final = tuple(family.name for family in _FAMILIES) +_STREAMS: Final = pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +_EACH_FAMILY: Final = pytest.mark.parametrize("family", _FAMILIES, ids=_FAMILY_IDS) + + +@dataclass(frozen=True, slots=True) +class _Route: + path: Callable[[str], str] + body: Callable[[str, str], dict[str, JsonValue]] + model: Callable[[_Rig], str] + + +_ROUTES: Final[Mapping[str, _Route]] = MappingProxyType( + { + "chat": _Route(lambda model: "/v1/chat/completions", lambda m, k: _chat_body(m, k, False), CHAT.model), + "chat_unversioned": _Route(lambda model: "/chat/completions", lambda m, k: _chat_body(m, k, False), CHAT.model), + "chat_azure_alias": _Route( + lambda model: f"/openai/deployments/{model}/chat/completions", + lambda m, k: _chat_body(m, k, False), + CHAT.model, + ), + "chat_stream": _Route(lambda model: "/v1/chat/completions", lambda m, k: _chat_body(m, k, True), CHAT.model), + "messages": _Route(lambda model: "/v1/messages", lambda m, k: _messages_body(m, k, False), MESSAGES.model), + "messages_stream": _Route( + lambda model: "/v1/messages", lambda m, k: _messages_body(m, k, True), MESSAGES.model + ), + "responses": _Route(lambda model: "/v1/responses", lambda m, k: _responses_body(m, k, False), RESPONSES.model), + "responses_stream": _Route( + lambda model: "/v1/responses", lambda m, k: _responses_body(m, k, True), RESPONSES.model + ), + "responses_compact": _Route( + lambda model: "/v1/responses/compact", lambda m, k: {"model": m, "input": k}, RESPONSES.model + ), + "generate_content": _Route( + lambda model: _generate_path(model, False), lambda m, k: _generate_body(m, k, False), GENERATE.model + ), + "stream_generate_content": _Route( + lambda model: _generate_path(model, True), lambda m, k: _generate_body(m, k, True), GENERATE.model + ), + "interactions": _Route( + lambda model: "/v1beta/interactions", lambda m, k: {"model": m, "input": k}, lambda rig: rig.gemini + ), + } +) +_MALFORMED_TOOLS: Final[Mapping[str, tuple[JsonValue, str]]] = MappingProxyType( + { + "int": (5, _NOT_A_LIST), + "empty_string": ("", _NOT_A_LIST), + "five_kb_string": ("x" * 5120, _NOT_A_LIST), + "list_of_names": (["get_weather"], _NOT_AN_OBJECT), + "empty_object": ({}, _NOT_A_LIST), + "false": (False, _NOT_A_LIST), + "zero": (0, _NOT_A_LIST), + } +) +_MALFORMED_ENTRIES: Final[Mapping[str, tuple[JsonValue, str]]] = MappingProxyType( + { + "int_entry": (5, _NOT_AN_OBJECT), + "string_function": ( + {"type": "function", "function": "get_weather"}, + "tools.0.function: Input should be a valid dictionary", + ), + "string_function_parameters": ( + {"type": "function", "function": {"name": "get_weather", "parameters": "bad"}}, + "tools.0.function.parameters: Input should be a valid dictionary", + ), + "list_input_schema": ( + {"name": "get_weather", "input_schema": [1]}, + "tools.0.input_schema: Input should be a valid dictionary", + ), + "int_parameters": ( + {"type": "function", "name": "get_weather", "parameters": 5}, + "tools.0.parameters: Input should be a valid dictionary", + ), + "string_vector_store_ids": ( + {"type": "file_search", "vector_store_ids": _OTHER_STORE}, + "tools.0.vector_store_ids: Input should be a valid list", + ), + } +) + + +def _send( + rig: _Rig, + family: _Family, + marker: str, + stream: bool, + *, + model: str | None = None, + key: str | None = None, + call_id: str | None = None, + user_agent: str | None = None, + **fields: JsonValue, +) -> httpx.Response: + chosen: Final = model or family.model(rig) + headers: Final = { + **({"x-litellm-call-id": call_id} if call_id else {}), + **({"user-agent": user_agent} if user_agent else {}), + } + return rig.gateway.request( + "POST", family.path(chosen, stream), {**family.body(chosen, marker, stream), **fields}, key=key, headers=headers + ) + + +def _delivered(rig: _Rig, marker: str) -> tuple[Request, ...]: + return tuple(request for request in rig.wire.drain() if marker in request.body.decode()) + + +def _one_delivery(rig: _Rig, marker: str, target: str) -> dict[str, JsonValue]: + (sent,) = _delivered(rig, marker) + assert sent.method == "POST" and target in sent.target, sent.target + return _JSON_OBJECT.validate_json(sent.body) + + +def _tool_names(family: _Family, body: Mapping[str, JsonValue]) -> tuple[str, ...]: + tools: Final = body.get("tools") + if not isinstance(tools, list): + return () + if family is GENERATE: + declarations: Final = chain.from_iterable(_declarations(object_value(tool)) for tool in tools) + return tuple(string_value(object_value(declaration)["name"]) for declaration in declarations) + if family is CHAT: + return tuple(string_value(object_value(object_value(tool)["function"])["name"]) for tool in tools) + return tuple(string_value(object_value(tool)["name"]) for tool in tools) + + +def _declarations(tool: Mapping[str, JsonValue]) -> list[JsonValue]: + declarations: Final = tool.get("functionDeclarations", tool.get("function_declarations")) + assert isinstance(declarations, list), tool + return declarations + + +def _assert_answered(family: _Family, response: httpx.Response, marker: str) -> None: + assert response.status_code == 200, response.text + tag: Final = _tag(marker) + assert _answer(tag) in response.text, response.text + upstream_id: Final = family.upstream_id(tag) + if upstream_id is not None: + assert upstream_id in response.text, response.text + if family is RESPONSES and not response.headers.get("content-type", "").startswith("text/event-stream"): + assert same_response(string_value(_JSON_OBJECT.validate_json(response.content)["id"]), f"resp_{tag}") + + +def _rejection(response: httpx.Response) -> tuple[str, str, JsonValue]: + body: Final = _JSON_OBJECT.validate_json(response.content) + error: Final = object_value(body["error"]) + if body.get("type") == "error": + return string_value(error["type"]), string_value(error["message"]), None + return string_value(error["type"]), string_value(error["message"]), error.get("param") + + +def _assert_rejected(response: httpx.Response, message: str, param: str = "tools") -> None: + assert response.status_code == 400, response.text + kind, said, named = _rejection(response) + assert kind == "invalid_request_error", response.text + assert message in said, response.text + assert named in (param, None), response.text + + +def _spend_rows(call_id: str) -> list[dict[str, JsonValue]]: + return read_rows(_SPEND_ROWS, (call_id, call_id)) + + +def _logged_once(call_id: str) -> dict[str, JsonValue]: + (row,) = held(lambda: _spend_rows(call_id), lambda rows: len(rows) == 1, holding=3, seconds=70) + return row + + +def _tool_values(sent: Mapping[str, JsonValue]) -> frozenset[str]: + return frozenset(name for name in _TOOL_FIELDS if sent.get(name) is not None) + + +def _call_id() -> str: + return f"malformed-tools-{uuid.uuid4().hex}" + + +@_STREAMS +@_EACH_FAMILY +def test_well_formed_tools_reach_the_provider_and_log_one_success_row(rig: _Rig, family: _Family, stream: bool) -> None: + marker: Final = _marker() + call_id: Final = _call_id() + response: Final = _send(rig, family, marker, stream, call_id=call_id, tools=[family.tool]) + _assert_answered(family, response, marker) + sent: Final = _one_delivery(rig, marker, family.target(stream)) + assert _tool_names(family, sent) == ("get_weather",), sent + if family.exact_tools: + assert sent["tools"] == [family.tool], sent + row: Final = _logged_once(call_id) + assert (row["call_type"], row["status"]) == (family.call_type(stream), "success"), row + + +def test_openai_sdk_chat_with_tools_answers_with_the_provider_reply(rig: _Rig) -> None: + marker: Final = _marker() + client: Final = openai.OpenAI( + base_url=f"{rig.gateway.client.base_url}/v1", + api_key=rig.gateway.key, + max_retries=0, + http_client=httpx.Client(trust_env=False), + ) + completion: Final = client.chat.completions.create( + model=rig.openai, + messages=[{"role": "user", "content": marker}], + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Look up the weather", + "parameters": _WEATHER_SCHEMA, + }, + } + ], + ) + assert completion.id == f"chatcmpl-{_tag(marker)}", completion + assert completion.choices[0].message.content == _answer(_tag(marker)), completion + assert _one_delivery(rig, marker, "/v1/chat/completions")["tools"] == [CHAT.tool] + + +def test_openai_sdk_rejects_malformed_tools_with_a_400(rig: _Rig) -> None: + marker: Final = _marker() + client: Final = openai.OpenAI( + base_url=f"{rig.gateway.client.base_url}/v1", + api_key=rig.gateway.key, + max_retries=0, + http_client=httpx.Client(trust_env=False), + ) + with pytest.raises(openai.BadRequestError) as raised: + client.chat.completions.create( + model=rig.openai, messages=[{"role": "user", "content": marker}], extra_body={"tools": 5} + ) + assert raised.value.status_code == 400, raised.value + assert object_value(raised.value.body)["param"] == "tools", raised.value.body + assert _delivered(rig, marker) == () + + +def test_async_openai_sdk_streams_responses_with_tools(rig: _Rig) -> None: + marker: Final = _marker() + + async def stream_text() -> str: + client: Final = openai.AsyncOpenAI( + base_url=f"{rig.gateway.client.base_url}/v1", + api_key=rig.gateway.key, + max_retries=0, + http_client=httpx.AsyncClient(trust_env=False), + ) + events: Final = await client.responses.create( + model=rig.openai, + input=marker, + stream=True, + tools=[ + { + "type": "function", + "name": "get_weather", + "description": "Look up the weather", + "parameters": _WEATHER_SCHEMA, + "strict": False, + } + ], + ) + completed: Final = [event async for event in events if event.type == "response.completed"] + assert len(completed) == 1, completed + return completed[0].response.output_text + + assert asyncio.run(stream_text()) == _answer(_tag(marker)) + assert _tool_names(RESPONSES, _one_delivery(rig, marker, "/v1/responses")) == ("get_weather",) + + +def test_anthropic_sdk_streams_messages_with_tools(rig: _Rig) -> None: + marker: Final = _marker() + client: Final = anthropic.Anthropic( + base_url=str(rig.gateway.client.base_url), + api_key=rig.gateway.key, + max_retries=0, + http_client=httpx.Client(trust_env=False), + ) + with client.messages.stream( + model=rig.anthropic, + max_tokens=16, + messages=[{"role": "user", "content": marker}], + tools=[{"name": "get_weather", "description": "Look up the weather", "input_schema": _WEATHER_SCHEMA}], + ) as stream: + final: Final = stream.get_final_message() + assert final.id == f"msg_{_tag(marker)}", final + assert _one_delivery(rig, marker, "/v1/messages")["tools"] == [MESSAGES.tool] + + +def test_google_genai_sdk_generates_with_tools(rig: _Rig) -> None: + marker: Final = _marker() + client: Final = genai.Client(api_key=rig.gateway.key, http_options={"base_url": str(rig.gateway.client.base_url)}) + response: Final = client.models.generate_content( + model=rig.gemini, + contents=marker, + config=types.GenerateContentConfig( + tools=[ + types.Tool( + function_declarations=[ + types.FunctionDeclaration( + name="get_weather", + description="Look up the weather", + parameters_json_schema=_WEATHER_SCHEMA, + ) + ] + ) + ], + automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True), + ), + ) + assert response.text == _answer(_tag(marker)), response + assert _tool_names(GENERATE, _one_delivery(rig, marker, ":generateContent")) == ("get_weather",) + + +@pytest.mark.parametrize("value", _MALFORMED_TOOLS.values(), ids=_MALFORMED_TOOLS.keys()) +@pytest.mark.parametrize("route", _ROUTES.values(), ids=_ROUTES.keys()) +def test_malformed_tools_answer_400_and_never_reach_the_provider( + rig: _Rig, route: _Route, value: tuple[JsonValue, str] +) -> None: + marker: Final = _marker() + model: Final = route.model(rig) + tools, message = value + response: Final = rig.gateway.request("POST", route.path(model), {**route.body(model, marker), "tools": tools}) + _assert_rejected(response, message) + assert _delivered(rig, marker) == () + + +@pytest.mark.parametrize("entry", _MALFORMED_ENTRIES.values(), ids=_MALFORMED_ENTRIES.keys()) +@pytest.mark.parametrize("family", (CHAT, MESSAGES), ids=("chat", "messages")) +def test_a_malformed_tool_entry_answers_400_naming_its_index( + rig: _Rig, family: _Family, entry: tuple[JsonValue, str] +) -> None: + marker: Final = _marker() + tool, message = entry + _assert_rejected(_send(rig, family, marker, False, tools=[tool]), message) + assert _delivered(rig, marker) == () + + +@pytest.mark.parametrize("value", [5, "x", []], ids=["int", "string", "list"]) +def test_malformed_web_search_options_answer_400(rig: _Rig, value: JsonValue) -> None: + marker: Final = _marker() + response: Final = _send(rig, CHAT, marker, False, web_search_options=value) + expected: Final = "web_search_options: Input should be a valid dictionary" + _assert_rejected(response, expected, param="web_search_options") + assert _delivered(rig, marker) == () + + +@pytest.mark.parametrize("value", [5, "x", []], ids=["int", "string", "list"]) +def test_malformed_web_search_options_drop_under_drop_params(rig: _Rig, value: JsonValue) -> None: + marker: Final = _marker() + response: Final = _send(rig, CHAT, marker, False, web_search_options=value, drop_params=True) + _assert_answered(CHAT, response, marker) + assert "web_search_options" not in _one_delivery(rig, marker, "/v1/chat/completions") + + +@_EACH_FAMILY +def test_wrong_key_with_malformed_tools_answers_401(rig: _Rig, family: _Family) -> None: + marker: Final = _marker() + response: Final = _send(rig, family, marker, False, key=f"sk-wrong-{uuid.uuid4().hex}", tools=5) + assert response.status_code == 401, response.text + assert _delivered(rig, marker) == () + + +def _count_body(rig: _Rig, marker: str, **fields: JsonValue) -> dict[str, JsonValue]: + return {"model": rig.anthropic, "messages": [{"role": "user", "content": marker}], **fields} + + +@pytest.mark.parametrize( + "fields", + [ + {"tools": 5}, + {"tools": ""}, + {"tools": "x" * 5120}, + {"tools": ["get_weather"]}, + {"messages": "x"}, + ], + ids=["int_tools", "empty_string_tools", "five_kb_tools", "list_of_names", "string_messages"], +) +def test_count_tokens_answers_an_anthropic_400_and_keeps_the_connection( + rig: _Rig, fields: dict[str, JsonValue] +) -> None: + marker: Final = _marker() + port: Final = rig.gateway.client.base_url.port + assert port is not None + connection: Final = http.client.HTTPConnection(rig.gateway.client.base_url.host, port, timeout=30) + headers: Final = {"Authorization": f"Bearer {rig.gateway.key}", "Content-Type": "application/json"} + try: + connection.request( + "POST", "/v1/messages/count_tokens", body=json.dumps(_count_body(rig, marker, **fields)), headers=headers + ) + counted: Final = connection.getresponse() + counted_body: Final = _JSON_OBJECT.validate_json(counted.read()) + socket_after_count: Final = connection.sock + connection.request( + "POST", "/v1/messages", body=json.dumps(_messages_body(rig.anthropic, marker, False)), headers=headers + ) + followed: Final = connection.getresponse() + followed_body: Final = followed.read().decode() + socket_after_follow_up: Final = connection.sock + finally: + connection.close() + assert counted.status == 400, counted_body + assert counted_body["type"] == "error", counted_body + error: Final = object_value(counted_body["error"]) + assert error["type"] == "invalid_request_error", counted_body + assert string_value(error["message"]).startswith(next(iter(fields))), counted_body + assert followed.status == 200, followed_body + assert f"msg_{_tag(marker)}" in followed_body, followed_body + assert socket_after_count is not None and socket_after_follow_up is socket_after_count + (sent,) = _delivered(rig, marker) + assert sent.target == "/v1/messages", sent.target + + +def test_anthropic_sdk_count_tokens_rejects_malformed_tools(rig: _Rig) -> None: + marker: Final = _marker() + client: Final = anthropic.Anthropic( + base_url=str(rig.gateway.client.base_url), + api_key=rig.gateway.key, + max_retries=0, + http_client=httpx.Client(trust_env=False), + ) + with pytest.raises(anthropic.BadRequestError) as raised: + client.messages.count_tokens( + model=rig.anthropic, messages=[{"role": "user", "content": marker}], extra_body={"tools": 5} + ) + assert raised.value.status_code == 400, raised.value + assert object_value(object_value(raised.value.body)["error"])["message"] == _NOT_A_LIST, raised.value.body + assert _delivered(rig, marker) == () + + +def test_count_tokens_forwards_well_formed_tools(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = rig.gateway.request( + "POST", "/v1/messages/count_tokens", _count_body(rig, marker, tools=[MESSAGES.tool]) + ) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content) == {"input_tokens": _COUNT}, response.text + assert _one_delivery(rig, marker, "/v1/messages/count_tokens")["tools"] == [MESSAGES.tool] + + +@pytest.mark.parametrize( + ("headers", "fields"), + [({}, {"drop_params": True}), ({"user-agent": _CLAUDE_CODE_AGENT}, {})], + ids=["drop_params", "claude_code_agent"], +) +def test_count_tokens_drops_only_the_bad_entry( + rig: _Rig, headers: dict[str, str], fields: dict[str, JsonValue] +) -> None: + marker: Final = _marker() + response: Final = rig.gateway.request( + "POST", + "/v1/messages/count_tokens", + _count_body(rig, marker, tools=[MESSAGES.tool, 5], **fields), + headers=headers, + ) + assert response.status_code == 200, response.text + assert _one_delivery(rig, marker, "/v1/messages/count_tokens")["tools"] == [MESSAGES.tool] + + +def test_responses_input_tokens_counts_well_formed_tools(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = rig.gateway.request( + "POST", "/v1/responses/input_tokens", {"model": rig.anthropic, "input": marker, "tools": [RESPONSES.tool]} + ) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content) == { + "object": "response.input_tokens", + "input_tokens": _COUNT, + }, response.text + sent: Final = _one_delivery(rig, marker, "/v1/messages/count_tokens") + assert "get_weather" in json.dumps(sent["tools"]), sent + + +def test_responses_input_tokens_rejects_malformed_tools(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = rig.gateway.request( + "POST", "/v1/responses/input_tokens", {"model": rig.anthropic, "input": marker, "tools": 5} + ) + _assert_rejected(response, _NOT_A_LIST) + assert _delivered(rig, marker) == () + + +def test_responses_input_tokens_drops_only_the_bad_entry(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = rig.gateway.request( + "POST", + "/v1/responses/input_tokens", + {"model": rig.anthropic, "input": marker, "tools": [RESPONSES.tool, 5], "drop_params": True}, + ) + assert response.status_code == 200, response.text + sent: Final = _one_delivery(rig, marker, "/v1/messages/count_tokens") + tools: Final = sent["tools"] + assert isinstance(tools, list) and len(tools) == 1 and "get_weather" in json.dumps(tools), sent + + +def _allowlisted_key(scenario: Scenario) -> str: + return scenario.key(metadata={"allowed_tools": ["get_weather"]}) + + +@_EACH_FAMILY +def test_allowlisted_key_keeps_its_allowlist_with_malformed_tools(rig: _Rig, family: _Family) -> None: + with rig.gateway.scenario() as scenario: + key: Final = _allowlisted_key(scenario) + allowed: Final = _marker() + _assert_answered(family, _send(rig, family, allowed, False, key=key, tools=[family.tool]), allowed) + assert len(_delivered(rig, allowed)) == 1 + malformed: Final = _marker() + _assert_rejected(_send(rig, family, malformed, False, key=key, tools=5), _NOT_A_LIST) + denied_tool: Final = json.loads(json.dumps(family.tool).replace("get_weather", "delete_files")) + denied: Final = _marker() + refused: Final = _send(rig, family, denied, False, key=key, tools=[denied_tool]) + assert refused.status_code == 403, refused.text + assert "delete_files" in refused.text, refused.text + assert _delivered(rig, malformed) + _delivered(rig, denied) == () + + +def _file_search(store_ids: JsonValue) -> dict[str, JsonValue]: + return {"type": "file_search", "vector_store_ids": store_ids} + + +def test_store_limited_key_searches_its_store_from_a_file_search_tool(rig: _Rig) -> None: + marker: Final = _marker() + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(object_permission={"vector_stores": [_CONFIG_STORE]}) + response: Final = _send(rig, CHAT, marker, False, key=key, tools=[_file_search([_CONFIG_STORE])]) + _assert_answered(CHAT, response, marker) + sent: Final = _one_delivery(rig, marker, "/v1/chat/completions") + assert "file_search" not in json.dumps(sent), sent + searches: Final = eventually( + lambda: tuple( + observation + for observation in upstream_observations(rig.gateway) + if observation["path"] == f"/vector_stores/{_CONFIG_STORE}/search" and marker in json.dumps(observation) + ), + lambda found: len(found) == 1, + ) + assert len(searches) == 1, searches + + +@pytest.mark.parametrize( + ("store_ids", "status", "named"), + [ + ([_OTHER_STORE], 401, _OTHER_STORE), + (_CONFIG_STORE, 400, "tools.0.vector_store_ids: Input should be a valid list"), + (_OTHER_STORE, 401, _OTHER_STORE), + ([5, _OTHER_STORE], 401, _OTHER_STORE), + ([5, _CONFIG_STORE], 400, "tools.0.vector_store_ids.0: Input should be a valid string"), + ], + ids=["other_store", "allowed_store_as_string", "other_store_as_string", "int_beside_other", "int_beside_allowed"], +) +def test_store_limited_key_reads_every_string_store_id( + rig: _Rig, store_ids: JsonValue, status: int, named: str +) -> None: + marker: Final = _marker() + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(object_permission={"vector_stores": [_CONFIG_STORE]}) + response: Final = _send(rig, CHAT, marker, False, key=key, tools=[_file_search(store_ids)]) + assert response.status_code == status, response.text + assert named in response.text, response.text + assert _delivered(rig, marker) == () + + +def test_store_limited_key_treats_null_store_ids_as_none(rig: _Rig) -> None: + marker: Final = _marker() + with rig.gateway.scenario() as scenario: + key: Final = scenario.key(object_permission={"vector_stores": [_CONFIG_STORE]}) + response: Final = _send(rig, CHAT, marker, False, key=key, tools=[_file_search(None)]) + _assert_answered(CHAT, response, marker) + assert _one_delivery(rig, marker, "/v1/chat/completions")["tools"] == [_file_search(None)] + + +@pytest.mark.parametrize( + ("first", "last", "status"), + [("5", "WELL_FORMED", 200), ("WELL_FORMED", "5", 400)], + ids=["well_formed_last", "malformed_last"], +) +def test_duplicate_tools_keys_use_the_last_one(rig: _Rig, first: str, last: str, status: int) -> None: + marker: Final = _marker() + well_formed: Final = json.dumps([CHAT.tool]) + prefix: Final = json.dumps({"model": rig.openai, "messages": [{"role": "user", "content": marker}]})[:-1] + raw: Final = ( + f'{prefix}, "tools": {first.replace("WELL_FORMED", well_formed)}, ' + f'"tools": {last.replace("WELL_FORMED", well_formed)}}}' + ) + response: Final = rig.gateway.client.post( + "/v1/chat/completions", + content=raw.encode(), + headers={"Authorization": f"Bearer {rig.gateway.key}", "Content-Type": "application/json"}, + ) + assert response.status_code == status, response.text + delivered: Final = _delivered(rig, marker) + if status == 400: + _assert_rejected(response, _NOT_A_LIST) + assert delivered == () + return + (sent,) = delivered + assert _JSON_OBJECT.validate_json(sent.body)["tools"] == [CHAT.tool] + + +@pytest.mark.parametrize( + ("family", "route", "expected"), + [ + (CHAT, None, "acompletion"), + (MESSAGES, None, "anthropic_messages"), + (RESPONSES, None, "aresponses"), + (GENERATE, None, "agenerate_content"), + (None, "interactions", "acreate_interaction"), + ], + ids=["chat", "messages", "responses", "generate_content", "interactions"], +) +def test_a_malformed_tools_rejection_logs_one_failure_row_with_its_call_type( + rig: _Rig, family: _Family | None, route: str | None, expected: str +) -> None: + marker: Final = _marker() + call_id: Final = _call_id() + response: Final = ( + _send(rig, family, marker, False, call_id=call_id, tools=5) + if family is not None + else rig.gateway.request( + "POST", + "/v1beta/interactions", + {"model": rig.gemini, "input": [{"type": "text", "text": marker}], "tools": 5}, + headers={"x-litellm-call-id": call_id}, + ) + ) + _assert_rejected(response, _NOT_A_LIST) + row: Final = _logged_once(call_id) + assert (row["call_type"], row["status"]) == (expected, "failure"), row + + +@pytest.mark.parametrize( + ("path", "body", "expected"), + [ + ("/v1/chat/completions", "chat", "acompletion"), + ("/v1/messages", "messages", "anthropic_messages"), + ("/v1beta/interactions", "interactions", "acreate_interaction"), + ], + ids=["chat", "messages", "interactions_list_input"], +) +def test_a_provider_401_with_well_formed_tools_logs_its_endpoint_call_type( + rig: _Rig, path: str, body: str, expected: str +) -> None: + marker: Final = f"{_marker()} {_REJECT_MARK}" + call_id: Final = _call_id() + payloads: Final[Mapping[str, dict[str, JsonValue]]] = { + "chat": {**_chat_body(rig.openai, marker, False), "tools": [CHAT.tool]}, + "messages": {**_messages_body(rig.anthropic, marker, False), "tools": [MESSAGES.tool]}, + "interactions": { + "model": rig.gemini, + "input": [{"type": "text", "text": marker}], + "tools": [{"type": "function", "name": "get_weather", "parameters": _WEATHER_SCHEMA}], + }, + } + response: Final = rig.gateway.request("POST", path, payloads[body], headers={"x-litellm-call-id": call_id}) + assert response.status_code == 401, response.text + assert len(_delivered(rig, marker)) == 1 + row: Final = _logged_once(call_id) + assert (row["call_type"], row["status"]) == (expected, "failure"), row + + +@pytest.mark.parametrize("drop", [True, "true"], ids=["bool", "string"]) +@_EACH_FAMILY +def test_request_drop_params_drops_malformed_tools_and_their_companions( + rig: _Rig, family: _Family, drop: JsonValue +) -> None: + marker: Final = _marker() + companion, value = family.tool_choice + response: Final = _send(rig, family, marker, False, tools=5, drop_params=drop, **{companion: value}) + _assert_answered(family, response, marker) + sent: Final = _one_delivery(rig, marker, family.target(False)) + assert _tool_values(sent) == frozenset(), sent + + +@_EACH_FAMILY +def test_drop_params_keeps_the_good_entries_and_the_tool_choice(rig: _Rig, family: _Family) -> None: + marker: Final = _marker() + companion, value = family.tool_choice + response: Final = _send(rig, family, marker, False, tools=[family.tool, 5], drop_params=True, **{companion: value}) + _assert_answered(family, response, marker) + sent: Final = _one_delivery(rig, marker, family.target(False)) + assert _tool_names(family, sent) == ("get_weather",), sent + if family.exact_tools: + assert sent["tools"] == [family.tool], sent + assert companion in sent, sent + + +def test_additional_drop_params_naming_tools_drops_them(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _send(rig, CHAT, marker, False, tools=5, additional_drop_params=["tools"]) + _assert_answered(CHAT, response, marker) + assert "tools" not in _one_delivery(rig, marker, "/v1/chat/completions") + + +def test_additional_drop_params_naming_another_field_still_rejects_tools(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _send(rig, CHAT, marker, False, tools=5, additional_drop_params=["web_search_options"]) + _assert_rejected(response, _NOT_A_LIST) + assert _delivered(rig, marker) == () + + +@pytest.mark.parametrize("family", (CHAT, GENERATE), ids=("chat", "generate_content")) +def test_a_drop_params_deployment_group_drops_malformed_tools(rig: _Rig, family: _Family) -> None: + marker: Final = _marker() + model: Final = rig.drop_group if family is CHAT else rig.gemini_drop_group + response: Final = _send(rig, family, marker, False, model=model, tools=5) + _assert_answered(family, response, marker) + assert _tool_values(_one_delivery(rig, marker, family.target(False))) == frozenset() + + +def test_a_drop_params_deployment_keeps_its_own_tools_when_it_drops_malformed_request_tools(rig: _Rig) -> None: + marker: Final = _marker() + companion, value = CHAT.tool_choice + response: Final = _send( + rig, CHAT, marker, False, model=rig.tools_drop_group, tools=5, parallel_tool_calls=False, **{companion: value} + ) + _assert_answered(CHAT, response, marker) + sent: Final = _one_delivery(rig, marker, "/v1/chat/completions") + assert _tool_names(CHAT, sent) == _tool_names(CHAT, {"tools": [CHAT.tool]}), sent + assert _tool_values(sent) == frozenset({"tools"}), sent + + +@pytest.mark.parametrize("drop", [False, None], ids=["false", "null"]) +def test_an_explicit_request_drop_params_overrides_the_deployment_group(rig: _Rig, drop: JsonValue) -> None: + marker: Final = _marker() + response: Final = _send(rig, CHAT, marker, False, model=rig.drop_group, tools=5, drop_params=drop) + _assert_rejected(response, _NOT_A_LIST) + assert _delivered(rig, marker) == () + + +def test_a_mixed_group_answers_malformed_tools_by_the_drop_params_of_the_deployment_it_picks(rig: _Rig) -> None: + markers: Final = tuple(_marker() for _ in range(20)) + statuses: Final = tuple( + _send(rig, CHAT, marker, False, model=rig.mixed_group, tools=5).status_code for marker in markers + ) + sent: Final = tuple(request.body.decode() for request in rig.wire.drain()) + deliveries: Final = tuple( + tuple(_JSON_OBJECT.validate_json(body) for body in sent if marker in body) for marker in markers + ) + assert sorted(set(statuses)) == [200, 400], statuses + assert all((status, len(bodies)) in ((200, 1), (400, 0)) for status, bodies in zip(statuses, deliveries)), ( + statuses, + deliveries, + ) + assert all(all("tools" not in body for body in bodies) for bodies in deliveries), deliveries + + +def test_a_deployment_dropping_tools_drops_every_tool(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _send(rig, CHAT, marker, False, model=rig.field_group, tools=[CHAT.tool, 5]) + _assert_answered(CHAT, response, marker) + assert "tools" not in _one_delivery(rig, marker, "/v1/chat/completions") + + +def test_a_request_additional_drop_params_list_keeps_tools_a_deployment_list_drops(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _send( + rig, + CHAT, + marker, + False, + model=rig.field_group, + tools=[CHAT.tool], + additional_drop_params=["web_search_options"], + ) + _assert_answered(CHAT, response, marker) + assert _tool_names(CHAT, _one_delivery(rig, marker, "/v1/chat/completions")) == ("get_weather",) + + +def test_a_request_additional_drop_params_list_replaces_the_deployment_list(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _send( + rig, + CHAT, + marker, + False, + model=rig.field_group, + tools=[CHAT.tool, 5], + additional_drop_params=["web_search_options"], + ) + _assert_rejected(response, _SECOND_NOT_AN_OBJECT) + assert _delivered(rig, marker) == () + + +@pytest.mark.parametrize( + ("family", "agent"), + [(MESSAGES, _CLAUDE_CODE_AGENT), (RESPONSES, _CODEX_AGENT), (CHAT, _CLAUDE_CODE_AGENT)], + ids=["claude_code_messages", "codex_responses", "claude_code_chat"], +) +def test_an_agentic_cli_drops_only_the_bad_entry(rig: _Rig, family: _Family, agent: str) -> None: + marker: Final = _marker() + response: Final = _send(rig, family, marker, False, user_agent=agent, tools=[family.tool, 5]) + _assert_answered(family, response, marker) + assert _one_delivery(rig, marker, family.target(False))["tools"] == [family.tool] + + +def test_an_agentic_cli_with_explicit_drop_params_false_rejects(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _send( + rig, MESSAGES, marker, False, user_agent=_CLAUDE_CODE_AGENT, tools=[MESSAGES.tool, 5], drop_params=False + ) + _assert_rejected(response, _SECOND_NOT_AN_OBJECT) + assert _delivered(rig, marker) == () + + +@pytest.mark.parametrize( + "fields", [{"tools": None}, {"tools": []}, {}], ids=["null_tools", "empty_list_tools", "missing_tools"] +) +def test_empty_null_and_missing_tools_still_answer(rig: _Rig, fields: dict[str, JsonValue]) -> None: + marker: Final = _marker() + response: Final = _send(rig, CHAT, marker, False, **fields) + _assert_answered(CHAT, response, marker) + assert not _one_delivery(rig, marker, "/v1/chat/completions").get("tools") + + +def test_null_web_search_options_answers_and_logs_success(rig: _Rig) -> None: + marker: Final = _marker() + call_id: Final = _call_id() + response: Final = _send(rig, CHAT, marker, False, call_id=call_id, web_search_options=None) + _assert_answered(CHAT, response, marker) + assert _one_delivery(rig, marker, "/v1/chat/completions").get("web_search_options") is None + row: Final = _logged_once(call_id) + assert (row["call_type"], row["status"]) == ("acompletion", "success"), row + + +@_EACH_FAMILY +def test_request_drop_params_with_well_formed_tools_logs_one_success_row(rig: _Rig, family: _Family) -> None: + marker: Final = _marker() + call_id: Final = _call_id() + response: Final = _send(rig, family, marker, False, call_id=call_id, tools=[family.tool], drop_params=True) + _assert_answered(family, response, marker) + assert _tool_names(family, _one_delivery(rig, marker, family.target(False))) == ("get_weather",) + row: Final = _logged_once(call_id) + assert (row["call_type"], row["status"]) == (family.call_type(False), "success"), row + + +def test_repeated_identical_dropped_requests_each_log_once(rig: _Rig) -> None: + marker: Final = _marker() + call_ids: Final = tuple(_call_id() for _ in range(3)) + for call_id in call_ids: + response = _send(rig, GENERATE, marker, False, call_id=call_id, tools=[GENERATE.tool, 5], drop_params=True) + _assert_answered(GENERATE, response, marker) + delivered: Final = _delivered(rig, marker) + assert ( + tuple(_tool_names(GENERATE, _JSON_OBJECT.validate_json(sent.body)) for sent in delivered) + == (("get_weather",),) * 3 + ) + rows: Final = tuple(_logged_once(call_id) for call_id in call_ids) + assert tuple((row["call_type"], row["status"]) for row in rows) == (("agenerate_content", "success"),) * 3 + + +@pytest.mark.parametrize("family", (CHAT, MESSAGES, RESPONSES), ids=("chat", "messages", "responses")) +def test_a_cache_hit_twin_answers_from_the_cache_and_logs_once_each(rig: _Rig, family: _Family) -> None: + marker: Final = _marker() + first_id: Final = _call_id() + second_id: Final = _call_id() + first: Final = _send(rig, family, marker, False, call_id=first_id, tools=[family.tool]) + _assert_answered(family, first, marker) + await_entry(redis_store(), _answer(_tag(marker))) + second: Final = _send(rig, family, marker, False, call_id=second_id, tools=[family.tool]) + _assert_answered(family, second, marker) + assert len(_delivered(rig, marker)) == 1 + first_row: Final = _logged_once(first_id) + second_row: Final = _logged_once(second_id) + assert (first_row["call_type"], second_row["call_type"]) == (family.call_type(False),) * 2, (first_row, second_row) + assert second_row["cache_hit"] == "True", second_row + + +def test_a_dropped_request_shares_the_cache_entry_of_its_clean_twin(rig: _Rig) -> None: + marker: Final = _marker() + dropped: Final = _send(rig, CHAT, marker, False, tools=[CHAT.tool, 5], drop_params=True) + _assert_answered(CHAT, dropped, marker) + await_entry(redis_store(), _answer(_tag(marker))) + clean: Final = _send(rig, CHAT, marker, False, tools=[CHAT.tool], drop_params=True) + _assert_answered(CHAT, clean, marker) + (sent,) = _delivered(rig, marker) + assert _JSON_OBJECT.validate_json(sent.body)["tools"] == [CHAT.tool] + + +def _field_group(rig: _Rig, groups: _FieldGroups, family: _Family) -> str: + return {MESSAGES.name: groups.anthropic, GENERATE.name: groups.gemini}.get(family.name, rig.field_group) + + +_FIELD_GROUP_FAMILIES: Final = pytest.mark.parametrize( + "family", (MESSAGES, RESPONSES, GENERATE), ids=("messages", "responses", "generate_content") +) + + +@_FIELD_GROUP_FAMILIES +def test_each_family_rejects_malformed_tools_when_a_request_list_replaces_the_deployment_list( + rig: _Rig, field_groups: _FieldGroups, family: _Family +) -> None: + marker: Final = _marker() + response: Final = _send( + rig, + family, + marker, + False, + model=_field_group(rig, field_groups, family), + tools=[family.tool, 5], + additional_drop_params=["web_search_options"], + ) + _assert_rejected(response, _SECOND_NOT_AN_OBJECT) + assert _delivered(rig, marker) == () + + +@pytest.mark.parametrize( + ("family", "forwarded"), + [(MESSAGES, ()), (RESPONSES, ()), (GENERATE, ("get_weather",))], + ids=("messages", "responses", "generate_content"), +) +def test_each_family_answers_malformed_tools_a_deployment_list_drops( + rig: _Rig, field_groups: _FieldGroups, family: _Family, forwarded: tuple[str, ...] +) -> None: + marker: Final = _marker() + response: Final = _send( + rig, family, marker, False, model=_field_group(rig, field_groups, family), tools=[family.tool, 5] + ) + _assert_answered(family, response, marker) + assert _tool_names(family, _one_delivery(rig, marker, family.target(False))) == forwarded + + +@_FIELD_GROUP_FAMILIES +def test_each_family_forwards_well_formed_tools_a_request_list_keeps( + rig: _Rig, field_groups: _FieldGroups, family: _Family +) -> None: + marker: Final = _marker() + response: Final = _send( + rig, + family, + marker, + False, + model=_field_group(rig, field_groups, family), + tools=[family.tool], + additional_drop_params=["web_search_options"], + ) + _assert_answered(family, response, marker) + assert _tool_names(family, _one_delivery(rig, marker, family.target(False))) == ("get_weather",) + + +_NOT_A_NAME_LIST: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "empty_list": [], + "null": None, + "string": "tools", + "empty_string": "", + "five_kb_string": "x" * 5120, + "int": 5, + } +) + + +@pytest.mark.parametrize("value", _NOT_A_NAME_LIST.values(), ids=_NOT_A_NAME_LIST.keys()) +def test_a_request_additional_drop_params_that_is_no_list_of_names_still_replaces_the_deployment_list( + rig: _Rig, value: JsonValue +) -> None: + marker: Final = _marker() + response: Final = _send( + rig, CHAT, marker, False, model=rig.field_group, tools=[CHAT.tool, 5], additional_drop_params=value + ) + _assert_rejected(response, _SECOND_NOT_AN_OBJECT) + assert _delivered(rig, marker) == () + + +@pytest.mark.parametrize( + "value", + [[], None, "tools", "", "x" * 5120], + ids=["empty_list", "null", "string", "empty_string", "five_kb_string"], +) +def test_a_request_additional_drop_params_replacing_the_deployment_list_keeps_well_formed_tools( + rig: _Rig, value: JsonValue +) -> None: + marker: Final = _marker() + response: Final = _send( + rig, CHAT, marker, False, model=rig.field_group, tools=[CHAT.tool], additional_drop_params=value + ) + _assert_answered(CHAT, response, marker) + assert _one_delivery(rig, marker, "/v1/chat/completions")["tools"] == [CHAT.tool] + + +@pytest.mark.parametrize( + ("first", "last", "status"), + [('["tools"]', '["web_search_options"]', 400), ('["web_search_options"]', '["tools"]', 200)], + ids=["other_field_last", "tools_last"], +) +def test_duplicate_additional_drop_params_keys_use_the_last_one(rig: _Rig, first: str, last: str, status: int) -> None: + marker: Final = _marker() + prefix: Final = json.dumps( + {"model": rig.openai, "messages": [{"role": "user", "content": marker}], "tools": [CHAT.tool, 5]} + )[:-1] + response: Final = rig.gateway.client.post( + "/v1/chat/completions", + content=f'{prefix}, "additional_drop_params": {first}, "additional_drop_params": {last}}}'.encode(), + headers={"Authorization": f"Bearer {rig.gateway.key}", "Content-Type": "application/json"}, + ) + assert response.status_code == status, response.text + if status == 400: + _assert_rejected(response, _SECOND_NOT_AN_OBJECT) + assert _delivered(rig, marker) == () + return + _assert_answered(CHAT, response, marker) + assert "tools" not in _one_delivery(rig, marker, "/v1/chat/completions") + + +def test_count_tokens_drops_the_bad_entry_a_deployment_list_names(rig: _Rig, field_groups: _FieldGroups) -> None: + marker: Final = _marker() + response: Final = rig.gateway.request( + "POST", + "/v1/messages/count_tokens", + {**_count_body(rig, marker, tools=[MESSAGES.tool, 5]), "model": field_groups.anthropic}, + ) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content) == {"input_tokens": _COUNT}, response.text + assert _one_delivery(rig, marker, "/v1/messages/count_tokens")["tools"] == [MESSAGES.tool] + + +def test_count_tokens_rejects_malformed_tools_when_a_request_list_replaces_the_deployment_list( + rig: _Rig, field_groups: _FieldGroups +) -> None: + marker: Final = _marker() + response: Final = rig.gateway.request( + "POST", + "/v1/messages/count_tokens", + { + **_count_body(rig, marker, tools=[MESSAGES.tool, 5], additional_drop_params=["web_search_options"]), + "model": field_groups.anthropic, + }, + ) + _assert_rejected(response, _SECOND_NOT_AN_OBJECT) + assert _delivered(rig, marker) == () + + +def test_responses_input_tokens_drops_the_bad_entry_a_deployment_list_names( + rig: _Rig, field_groups: _FieldGroups +) -> None: + marker: Final = _marker() + response: Final = rig.gateway.request( + "POST", + "/v1/responses/input_tokens", + {"model": field_groups.anthropic, "input": marker, "tools": [RESPONSES.tool, 5]}, + ) + assert response.status_code == 200, response.text + tools: Final = _one_delivery(rig, marker, "/v1/messages/count_tokens")["tools"] + assert isinstance(tools, list) and len(tools) == 1 and "get_weather" in json.dumps(tools), tools + + +def test_responses_input_tokens_rejects_malformed_tools_when_a_request_list_replaces_the_deployment_list( + rig: _Rig, field_groups: _FieldGroups +) -> None: + marker: Final = _marker() + response: Final = rig.gateway.request( + "POST", + "/v1/responses/input_tokens", + { + "model": field_groups.anthropic, + "input": marker, + "tools": [RESPONSES.tool, 5], + "additional_drop_params": ["web_search_options"], + }, + ) + _assert_rejected(response, _SECOND_NOT_AN_OBJECT) + assert _delivered(rig, marker) == () + + +def _text_items(marker: str) -> list[JsonValue]: + return [{"role": "user", "content": marker}] + + +@pytest.mark.parametrize("compact_input", [_text_items, lambda marker: marker], ids=["list_input", "string_input"]) +def test_a_compact_rejection_logs_one_failure_row_as_compact( + rig: _Rig, compact_input: Callable[[str], JsonValue] +) -> None: + marker: Final = _marker() + call_id: Final = _call_id() + response: Final = rig.gateway.request( + "POST", + "/v1/responses/compact", + {"model": rig.openai, "input": compact_input(marker), "tools": 5}, + headers={"x-litellm-call-id": call_id}, + ) + _assert_rejected(response, _NOT_A_LIST) + assert _delivered(rig, marker) == () + row: Final = _logged_once(call_id) + assert (row["call_type"], row["status"]) == ("acompact_responses", "failure"), row + + +@dataclass(frozen=True, slots=True) +class _Limited: + path: Callable[[_Rig], str] + body: Callable[[_Rig, str], dict[str, JsonValue]] + call_type: str + + +_LIMITED: Final[Mapping[str, _Limited]] = MappingProxyType( + { + "chat": _Limited( + lambda rig: "/v1/chat/completions", lambda rig, k: _chat_body(rig.openai, k, False), "acompletion" + ), + "messages": _Limited( + lambda rig: "/v1/messages", lambda rig, k: _messages_body(rig.anthropic, k, False), "anthropic_messages" + ), + "responses_list_input": _Limited( + lambda rig: "/v1/responses", lambda rig, k: {"model": rig.openai, "input": _text_items(k)}, "aresponses" + ), + "compact_list_input": _Limited( + lambda rig: "/v1/responses/compact", + lambda rig, k: {"model": rig.openai, "input": _text_items(k)}, + "acompact_responses", + ), + "generate_content": _Limited( + lambda rig: _generate_path(rig.gemini, False), + lambda rig, k: _generate_body(rig.gemini, k, False), + "agenerate_content", + ), + "interactions_list_input": _Limited( + lambda rig: "/v1beta/interactions", + lambda rig, k: {"model": rig.gemini, "input": [{"type": "text", "text": k}]}, + "acreate_interaction", + ), + "video_generation": _Limited( + lambda rig: "/v1/videos", lambda rig, k: {"model": rig.openai, "prompt": k}, "avideo_generation" + ), + } +) + + +@pytest.mark.parametrize("limited", _LIMITED.values(), ids=_LIMITED.keys()) +def test_a_rate_limited_request_logs_one_failure_row_with_its_endpoint_call_type( + rig: _Rig, limited_key: str, limited: _Limited +) -> None: + marker: Final = _marker() + call_id: Final = _call_id() + response: Final = rig.gateway.request( + "POST", + limited.path(rig), + limited.body(rig, marker), + key=limited_key, + headers={"x-litellm-call-id": call_id}, + ) + assert response.status_code == 429, response.text + assert _delivered(rig, marker) == () + row: Final = _logged_once(call_id) + assert (row["call_type"], row["status"]) == (limited.call_type, "failure"), row + + +@pytest.mark.parametrize("family", (CHAT, MESSAGES), ids=("chat", "messages")) +def test_a_rate_limited_twin_of_a_cached_request_logs_its_endpoint_call_type( + rig: _Rig, limited_key: str, family: _Family +) -> None: + marker: Final = _marker() + _assert_answered(family, _send(rig, family, marker, False, tools=[family.tool]), marker) + await_entry(redis_store(), _answer(_tag(marker))) + call_id: Final = _call_id() + response: Final = _send(rig, family, marker, False, key=limited_key, call_id=call_id, tools=[family.tool]) + assert response.status_code == 429, response.text + assert len(_delivered(rig, marker)) == 1 + row: Final = _logged_once(call_id) + assert (row["call_type"], row["status"]) == (family.call_type(False), "failure"), row + + +def test_a_rate_limited_image_edit_logs_one_failure_row_as_an_image_edit(rig: _Rig, limited_key: str) -> None: + marker: Final = _marker() + call_id: Final = _call_id() + response: Final = rig.gateway.client.post( + "/v1/images/edits", + data={"model": rig.openai, "prompt": marker}, + files={"image": ("edit.png", b"\x89PNG\r\n\x1a\n", "image/png")}, + headers={"Authorization": f"Bearer {limited_key}", "x-litellm-call-id": call_id}, + ) + assert response.status_code == 429, response.text + assert _delivered(rig, marker) == () + row: Final = _logged_once(call_id) + assert (row["call_type"], row["status"]) == ("aimage_edit", "failure"), row + + +_Sender = Callable[[_Rig, str, str, Mapping[str, JsonValue]], httpx.Response] + + +@dataclass(frozen=True, slots=True) +class _Checked: + provider: str + tool: dict[str, JsonValue] + named_as: _Family + send: _Sender + target: str + answered: Callable[[httpx.Response, str], None] + kept_by_a_tools_list: tuple[str, ...] + bad_entry: dict[str, JsonValue] + + +def _family_sender(family: _Family) -> _Sender: + return lambda rig, model, marker, fields: _send(rig, family, marker, False, model=model, **fields) + + +def _family_answered(family: _Family) -> Callable[[httpx.Response, str], None]: + return lambda response, marker: _assert_answered(family, response, marker) + + +def _count_tokens(rig: _Rig, model: str, marker: str, fields: Mapping[str, JsonValue]) -> httpx.Response: + return rig.gateway.request( + "POST", + "/v1/messages/count_tokens", + {"model": model, "messages": [{"role": "user", "content": marker}], **fields}, + ) + + +def _input_tokens(rig: _Rig, model: str, marker: str, fields: Mapping[str, JsonValue]) -> httpx.Response: + return rig.gateway.request("POST", "/v1/responses/input_tokens", {"model": model, "input": marker, **fields}) + + +def _counted(expected: Mapping[str, JsonValue]) -> Callable[[httpx.Response, str], None]: + def check(response: httpx.Response, marker: str) -> None: + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content) == expected, (marker, response.text) + + return check + + +_CHAT_BAD_ENTRY: Final[dict[str, JsonValue]] = { + "type": "function", + "function": {"name": "get_weather", "parameters": "bad"}, +} +_MESSAGES_BAD_ENTRY: Final[dict[str, JsonValue]] = {"name": "get_weather", "input_schema": [1]} +_RESPONSES_BAD_ENTRY: Final[dict[str, JsonValue]] = {"type": "function", "name": "get_weather", "parameters": 5} +_CHECKED: Final[Mapping[str, _Checked]] = MappingProxyType( + { + "chat": _Checked( + provider="openai", + tool=CHAT.tool, + named_as=CHAT, + send=_family_sender(CHAT), + target="/v1/chat/completions", + answered=_family_answered(CHAT), + kept_by_a_tools_list=(), + bad_entry=_CHAT_BAD_ENTRY, + ), + "messages": _Checked( + provider="anthropic", + tool=MESSAGES.tool, + named_as=MESSAGES, + send=_family_sender(MESSAGES), + target="/v1/messages", + answered=_family_answered(MESSAGES), + kept_by_a_tools_list=(), + bad_entry=_MESSAGES_BAD_ENTRY, + ), + "responses": _Checked( + provider="openai", + tool=RESPONSES.tool, + named_as=RESPONSES, + send=_family_sender(RESPONSES), + target="/v1/responses", + answered=_family_answered(RESPONSES), + kept_by_a_tools_list=(), + bad_entry=_RESPONSES_BAD_ENTRY, + ), + "generate_content": _Checked( + provider="gemini", + tool=GENERATE.tool, + named_as=GENERATE, + send=_family_sender(GENERATE), + target=":generateContent", + answered=_family_answered(GENERATE), + kept_by_a_tools_list=("get_weather",), + bad_entry={"parameters": 5}, + ), + "count_tokens": _Checked( + provider="anthropic", + tool=MESSAGES.tool, + named_as=MESSAGES, + send=_count_tokens, + target="/v1/messages/count_tokens", + answered=_counted({"input_tokens": _COUNT}), + kept_by_a_tools_list=("get_weather",), + bad_entry=_MESSAGES_BAD_ENTRY, + ), + "input_tokens": _Checked( + provider="anthropic", + tool=RESPONSES.tool, + named_as=MESSAGES, + send=_input_tokens, + target="/v1/messages/count_tokens", + answered=_counted({"object": "response.input_tokens", "input_tokens": _COUNT}), + kept_by_a_tools_list=("get_weather",), + bad_entry=_RESPONSES_BAD_ENTRY, + ), + } +) +_EACH_CHECKED: Final = pytest.mark.parametrize("checked", _CHECKED.values(), ids=_CHECKED.keys()) +_CHAT_AND_COUNT: Final = pytest.mark.parametrize( + "checked", (_CHECKED["chat"], _CHECKED["count_tokens"]), ids=("chat", "count_tokens") +) +_ENTRY_SHAPES: Final = pytest.mark.parametrize( + "bad_entry", + [lambda checked: 5, lambda checked: checked.bad_entry], + ids=["int_entry", "dict_entry"], +) +_BadEntry = Callable[[_Checked], JsonValue] + + +def _assert_forwarded( + rig: _Rig, checked: _Checked, response: httpx.Response, marker: str, names: tuple[str, ...] +) -> None: + checked.answered(response, marker) + assert _tool_names(checked.named_as, _one_delivery(rig, marker, checked.target)) == names + + +def _assert_held_back(rig: _Rig, response: httpx.Response, marker: str) -> None: + _assert_rejected(response, _SECOND_NOT_AN_OBJECT) + assert _delivered(rig, marker) == () + + +_PROVIDERS: Final[Mapping[str, Callable[[str], dict[str, JsonValue]]]] = MappingProxyType( + { + "openai": lambda wire: { + "model": f"openai/{_OPENAI_BACKEND}", + "api_base": f"{wire}/v1", + "api_key": _PROVIDER_KEY, + }, + "anthropic": lambda wire: {"model": f"anthropic/{_CLAUDE_BACKEND}", "api_base": wire, "api_key": _PROVIDER_KEY}, + "gemini": lambda wire: {"model": f"gemini/{_GEMINI_BACKEND}", "api_base": wire, "api_key": _PROVIDER_KEY}, + } +) +_DROP_VARIANTS: Final[Mapping[str, dict[str, JsonValue]]] = MappingProxyType( + { + "plain": {}, + "lists_web_search": {"additional_drop_params": ["web_search_options"]}, + "lists_tools": {"additional_drop_params": ["tools"]}, + "keeps": {"drop_params": False}, + } +) + + +@dataclass(frozen=True, slots=True) +class _Owned: + rig: _Rig + suffix: str + + +def _owned_name(suffix: str, variant: str, provider: str) -> str: + return f"malformed-tools-{variant}-{provider}-{suffix}" + + +def _owned_id(suffix: str, variant: str, provider: str) -> str: + return f"{_owned_name(suffix, variant, provider)}-id" + + +@dataclass(frozen=True, slots=True) +class _OwnedSetup: + router_settings: Callable[[str], dict[str, JsonValue]] + general: Mapping[str, JsonValue] = MappingProxyType({}) + extra_models: Callable[[str, str], tuple[dict[str, JsonValue], ...]] = lambda wire, suffix: () + + +def _router_defaults(defaults: dict[str, JsonValue]) -> _OwnedSetup: + return _OwnedSetup(router_settings=lambda suffix: {"default_litellm_params": defaults}) + + +def _owned_config(path: Path, wire: str, suffix: str, setup: _OwnedSetup) -> Path: + path.write_text( + json.dumps( + { + "model_list": [ + *( + { + "model_name": _owned_name(suffix, variant, provider), + "litellm_params": {**deployment(wire), **extra}, + "model_info": {"id": _owned_id(suffix, variant, provider)}, + } + for (variant, extra), (provider, deployment) in product( + _DROP_VARIANTS.items(), _PROVIDERS.items() + ) + ), + *setup.extra_models(wire, suffix), + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + **setup.general, + }, + "router_settings": setup.router_settings(suffix), + } + ) + ) + return path + + +@contextmanager +def _owned_proxy(rig: _Rig, directory: Path, setup: _OwnedSetup) -> Iterator[_Owned]: + suffix: Final = uuid.uuid4().hex[:12] + config: Final = _owned_config(directory / "owned.yaml", rig.wire.url, suffix, setup) + with owned_proxy_process(rig.gateway, directory, {}, config=config, workers=2) as proxy: + yield _Owned(replace(rig, gateway=proxy.gateway), suffix) + + +@pytest.fixture(scope="module") +def tools_default(rig: _Rig, tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Owned]: + directory: Final = tmp_path_factory.mktemp("malformed-tools-tools-default") + with _owned_proxy(rig, directory, _router_defaults({"additional_drop_params": ["tools"]})) as owned: + yield owned + + +@pytest.fixture(scope="module") +def drop_default(rig: _Rig, tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Owned]: + directory: Final = tmp_path_factory.mktemp("malformed-tools-drop-default") + with _owned_proxy(rig, directory, _router_defaults({"drop_params": True, "additional_drop_params": None})) as owned: + yield owned + + +@pytest.fixture(scope="module") +def client_side(rig: _Rig, tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Owned]: + directory: Final = tmp_path_factory.mktemp("malformed-tools-client-side") + setup: Final = replace( + _router_defaults({"additional_drop_params": "tools"}), + general=MappingProxyType({"allow_client_side_credentials": True}), + ) + with _owned_proxy(rig, directory, setup) as owned: + yield owned + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@_ENTRY_SHAPES +@_EACH_CHECKED +def test_a_router_default_list_naming_tools_answers_malformed_tools( + tools_default: _Owned, checked: _Checked, bad_entry: _BadEntry +) -> None: + marker: Final = _marker() + model: Final = _owned_name(tools_default.suffix, "plain", checked.provider) + response: Final = checked.send(tools_default.rig, model, marker, {"tools": [checked.tool, bad_entry(checked)]}) + _assert_forwarded(tools_default.rig, checked, response, marker, checked.kept_by_a_tools_list) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@_EACH_CHECKED +def test_a_request_list_replaces_a_router_default_list(tools_default: _Owned, checked: _Checked) -> None: + marker: Final = _marker() + model: Final = _owned_name(tools_default.suffix, "plain", checked.provider) + response: Final = checked.send( + tools_default.rig, + model, + marker, + {"tools": [checked.tool, 5], "additional_drop_params": ["web_search_options"]}, + ) + _assert_held_back(tools_default.rig, response, marker) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@_ENTRY_SHAPES +@_EACH_CHECKED +def test_a_router_default_list_replaces_a_deployment_list( + tools_default: _Owned, checked: _Checked, bad_entry: _BadEntry +) -> None: + marker: Final = _marker() + model: Final = _owned_name(tools_default.suffix, "lists_web_search", checked.provider) + response: Final = checked.send(tools_default.rig, model, marker, {"tools": [checked.tool, bad_entry(checked)]}) + _assert_forwarded(tools_default.rig, checked, response, marker, checked.kept_by_a_tools_list) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@_ENTRY_SHAPES +def test_a_router_default_list_replaces_the_list_of_a_deployment_named_by_id( + tools_default: _Owned, bad_entry: _BadEntry +) -> None: + marker: Final = _marker() + checked: Final = _CHECKED["chat"] + model: Final = _owned_id(tools_default.suffix, "lists_web_search", checked.provider) + response: Final = checked.send(tools_default.rig, model, marker, {"tools": [checked.tool, bad_entry(checked)]}) + _assert_forwarded(tools_default.rig, checked, response, marker, ()) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@pytest.mark.parametrize("value", [None, [], 5], ids=["null", "empty_list", "int"]) +def test_a_request_list_naming_nothing_replaces_a_router_default_list(tools_default: _Owned, value: JsonValue) -> None: + marker: Final = _marker() + model: Final = _owned_name(tools_default.suffix, "plain", "openai") + response: Final = _send( + tools_default.rig, CHAT, marker, False, model=model, tools=[CHAT.tool, 5], additional_drop_params=value + ) + _assert_held_back(tools_default.rig, response, marker) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_a_router_default_list_naming_tools_drops_well_formed_tools(tools_default: _Owned) -> None: + marker: Final = _marker() + model: Final = _owned_name(tools_default.suffix, "plain", "openai") + response: Final = _send(tools_default.rig, CHAT, marker, False, model=model, tools=[CHAT.tool]) + _assert_forwarded(tools_default.rig, _CHECKED["chat"], response, marker, ()) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@_ENTRY_SHAPES +@_EACH_CHECKED +def test_a_router_default_drop_params_keeps_the_good_entries( + drop_default: _Owned, checked: _Checked, bad_entry: _BadEntry +) -> None: + marker: Final = _marker() + model: Final = _owned_name(drop_default.suffix, "plain", checked.provider) + response: Final = checked.send(drop_default.rig, model, marker, {"tools": [checked.tool, bad_entry(checked)]}) + _assert_forwarded(drop_default.rig, checked, response, marker, ("get_weather",)) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@pytest.mark.parametrize("drop", [False, None], ids=["false", "null"]) +def test_a_request_drop_params_replaces_a_router_default_drop_params(drop_default: _Owned, drop: JsonValue) -> None: + marker: Final = _marker() + model: Final = _owned_name(drop_default.suffix, "plain", "openai") + response: Final = _send(drop_default.rig, CHAT, marker, False, model=model, tools=[CHAT.tool, 5], drop_params=drop) + _assert_held_back(drop_default.rig, response, marker) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@_ENTRY_SHAPES +@_CHAT_AND_COUNT +def test_a_router_default_drop_params_replaces_a_deployment_drop_params( + drop_default: _Owned, checked: _Checked, bad_entry: _BadEntry +) -> None: + marker: Final = _marker() + model: Final = _owned_name(drop_default.suffix, "keeps", checked.provider) + response: Final = checked.send(drop_default.rig, model, marker, {"tools": [checked.tool, bad_entry(checked)]}) + _assert_forwarded(drop_default.rig, checked, response, marker, ("get_weather",)) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@_ENTRY_SHAPES +@_CHAT_AND_COUNT +def test_a_null_router_default_list_leaves_the_deployment_list_in_force( + drop_default: _Owned, checked: _Checked, bad_entry: _BadEntry +) -> None: + marker: Final = _marker() + model: Final = _owned_name(drop_default.suffix, "lists_tools", checked.provider) + response: Final = checked.send( + drop_default.rig, model, marker, {"tools": [checked.tool, bad_entry(checked)], "drop_params": False} + ) + _assert_forwarded(drop_default.rig, checked, response, marker, checked.kept_by_a_tools_list) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_a_router_default_list_that_is_a_string_still_rejects_a_bad_tool_entry(client_side: _Owned) -> None: + marker: Final = _marker() + model: Final = _owned_name(client_side.suffix, "plain", "openai") + response: Final = _send(client_side.rig, CHAT, marker, False, model=model, tools=[CHAT.tool, 5]) + _assert_held_back(client_side.rig, response, marker) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_a_router_default_list_that_is_a_string_keeps_well_formed_tools(client_side: _Owned) -> None: + marker: Final = _marker() + model: Final = _owned_name(client_side.suffix, "plain", "openai") + response: Final = _send(client_side.rig, CHAT, marker, False, model=model, tools=[CHAT.tool]) + _assert_answered(CHAT, response, marker) + assert _one_delivery(client_side.rig, marker, "/v1/chat/completions")["tools"] == [CHAT.tool] + + +_USER_CONFIG_DROPS: Final[Mapping[str, tuple[dict[str, JsonValue], dict[str, JsonValue], tuple[str, ...]]]] = ( + MappingProxyType( + { + "deployment_list": ({"additional_drop_params": ["tools"]}, {}, ()), + "router_default_list": ({}, {"additional_drop_params": ["tools"]}, ()), + "router_default_drop_params": ({}, {"drop_params": True}, ("get_weather",)), + } + ) +) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@pytest.mark.parametrize(("deployment", "defaults", "kept"), _USER_CONFIG_DROPS.values(), ids=_USER_CONFIG_DROPS.keys()) +def test_a_user_config_router_decides_which_malformed_tools_drop( + client_side: _Owned, + deployment: dict[str, JsonValue], + defaults: dict[str, JsonValue], + kept: tuple[str, ...], +) -> None: + marker: Final = _marker() + checked: Final = _CHECKED["chat"] + name: Final = f"malformed-tools-user-config-{uuid.uuid4().hex}" + user_config: Final[dict[str, JsonValue]] = { + "model_list": [ + {"model_name": name, "litellm_params": {**_PROVIDERS["openai"](client_side.rig.wire.url), **deployment}} + ], + "default_litellm_params": defaults, + } + response: Final = checked.send( + client_side.rig, name, marker, {"tools": [checked.tool, checked.bad_entry], "user_config": user_config} + ) + _assert_forwarded(client_side.rig, checked, response, marker, kept) + + +@dataclass(frozen=True, slots=True) +class _ById: + lists_tools: Mapping[str, str] + drops: str + mixed_drops: str + mixed_keeps: str + shadowed: str + + +@pytest.fixture(scope="module") +def by_id(rig: _Rig) -> Iterator[_ById]: + openai_deployment: Final = _PROVIDERS["openai"](rig.wire.url) + lists_tools: Final[dict[str, JsonValue]] = {"additional_drop_params": ["tools"]} + names: Final = MappingProxyType({provider: f"integration-by-id-{uuid.uuid4().hex}" for provider in _PROVIDERS}) + drops_name: Final = f"integration-by-id-drops-{uuid.uuid4().hex}" + mixed: Final = f"integration-by-id-mixed-{uuid.uuid4().hex}" + shadow_name: Final = f"integration-by-id-shadow-{uuid.uuid4().hex}" + with rig.gateway.scenario() as scenario: + listed: Final = MappingProxyType( + { + provider: _deployment_ids(scenario, names[provider], ({**deployment(rig.wire.url), **lists_tools},))[0] + for provider, deployment in _PROVIDERS.items() + } + ) + (drops,) = _deployment_ids(scenario, drops_name, ({**openai_deployment, "drop_params": True},)) + mixed_drops, mixed_keeps = _deployment_ids( + scenario, mixed, ({**openai_deployment, **lists_tools}, openai_deployment) + ) + (shadowed,) = _deployment_ids(scenario, shadow_name, ({**openai_deployment, **lists_tools},)) + _deployment_ids(scenario, shadowed, (openai_deployment,)) + for group, deployments in ( + *((name, 1) for name in names.values()), + (drops_name, 1), + (mixed, 2), + (shadow_name, 1), + (shadowed, 1), + ): + wait_for_worker_readiness(rig.gateway, group, deployments, _PROXY_WORKERS) + yield _ById(listed, drops, mixed_drops, mixed_keeps, shadowed) + + +@_ENTRY_SHAPES +@_EACH_CHECKED +def test_a_deployment_id_answers_malformed_tools_its_deployment_list_drops( + rig: _Rig, by_id: _ById, checked: _Checked, bad_entry: _BadEntry +) -> None: + marker: Final = _marker() + response: Final = checked.send( + rig, by_id.lists_tools[checked.provider], marker, {"tools": [checked.tool, bad_entry(checked)]} + ) + _assert_forwarded(rig, checked, response, marker, checked.kept_by_a_tools_list) + + +@_EACH_CHECKED +def test_a_request_list_replaces_the_list_of_a_deployment_named_by_id( + rig: _Rig, by_id: _ById, checked: _Checked +) -> None: + marker: Final = _marker() + response: Final = checked.send( + rig, + by_id.lists_tools[checked.provider], + marker, + {"tools": [checked.tool, 5], "additional_drop_params": ["web_search_options"]}, + ) + _assert_held_back(rig, response, marker) + + +@_ENTRY_SHAPES +def test_the_id_of_the_dropping_deployment_in_a_mixed_group_drops_malformed_tools( + rig: _Rig, by_id: _ById, bad_entry: _BadEntry +) -> None: + marker: Final = _marker() + checked: Final = _CHECKED["chat"] + response: Final = checked.send(rig, by_id.mixed_drops, marker, {"tools": [checked.tool, bad_entry(checked)]}) + _assert_forwarded(rig, checked, response, marker, ()) + + +def test_the_keeping_id_of_a_mixed_group_rejects_malformed_tools(rig: _Rig, by_id: _ById) -> None: + marker: Final = _marker() + response: Final = _send(rig, CHAT, marker, False, model=by_id.mixed_keeps, tools=[CHAT.tool, 5]) + _assert_held_back(rig, response, marker) + + +@_ENTRY_SHAPES +def test_the_id_of_a_drop_params_deployment_keeps_the_good_entries( + rig: _Rig, by_id: _ById, bad_entry: _BadEntry +) -> None: + marker: Final = _marker() + checked: Final = _CHECKED["chat"] + response: Final = checked.send(rig, by_id.drops, marker, {"tools": [checked.tool, bad_entry(checked)]}) + _assert_forwarded(rig, checked, response, marker, ("get_weather",)) + + +def test_a_model_name_wins_over_an_equal_deployment_id(rig: _Rig, by_id: _ById) -> None: + marker: Final = _marker() + response: Final = _send(rig, CHAT, marker, False, model=by_id.shadowed, tools=[CHAT.tool, 5]) + _assert_held_back(rig, response, marker) + + +def test_a_deployment_id_with_a_tools_list_drops_well_formed_tools(rig: _Rig, by_id: _ById) -> None: + marker: Final = _marker() + response: Final = _send(rig, CHAT, marker, False, model=by_id.lists_tools["openai"], tools=[CHAT.tool]) + _assert_forwarded(rig, _CHECKED["chat"], response, marker, ()) + + +def _failed_over(request: Request) -> bool: + return request.target.startswith((_DOWN, *(prefix for prefix, _ in _REFUSALS.values()))) + + +def _assert_failed_over( + rig: _Rig, checked: _Checked, response: httpx.Response, marker: str, names: tuple[str, ...] +) -> None: + checked.answered(response, marker) + delivered: Final = _delivered(rig, marker) + (served,) = tuple(request for request in delivered if not _failed_over(request)) + assert checked.target in served.target, served.target + assert _tool_names(checked.named_as, _JSON_OBJECT.validate_json(served.body)) == names + + +_FALLBACK_CHECKED: Final = pytest.mark.parametrize( + "checked", + tuple(_CHECKED[name] for name in ("chat", "messages", "responses")), + ids=("chat", "messages", "responses"), +) +_TOKEN_COUNTS: Final = pytest.mark.parametrize( + "checked", (_CHECKED["count_tokens"], _CHECKED["input_tokens"]), ids=("count_tokens", "input_tokens") +) + + +@dataclass(frozen=True, slots=True) +class _Reroute: + down: Mapping[str, str] + lists_tools: Mapping[str, str] + refusing: Mapping[str, str] + fallback_keys: Mapping[str, str] + team_key: str + team_models: Mapping[str, str] + + +@pytest.fixture(scope="module") +def reroute(rig: _Rig, field_groups: _FieldGroups) -> Iterator[_Reroute]: + lists_tools: Final = MappingProxyType({"openai": rig.field_group, "anthropic": field_groups.anthropic}) + with rig.gateway.scenario() as scenario: + down: Final = MappingProxyType( + {provider: scenario.model(**_PROVIDERS[provider](f"{rig.wire.url}/down")) for provider in lists_tools} + ) + refusing: Final = MappingProxyType( + { + chain: scenario.model(**_PROVIDERS["openai"](f"{rig.wire.url}{prefix.rstrip('/')}")) + for chain, (prefix, _) in _REFUSALS.items() + } + ) + router_settings: Final[dict[str, JsonValue]] = {"fallbacks": [{down["openai"]: [rig.field_group]}]} + public_team: Final = scenario.team() + team_models: Final = MappingProxyType( + { + provider: scenario.model( + **_PROVIDERS[provider](rig.wire.url), + additional_drop_params=["tools"], + model_info={"team_id": public_team}, + ) + for provider in lists_tools + } + ) + built: Final = _Reroute( + down=down, + lists_tools=lists_tools, + refusing=refusing, + fallback_keys=MappingProxyType( + { + "key": scenario.key(router_settings=router_settings), + "team": scenario.key(team_id=scenario.team(router_settings=router_settings)), + } + ), + team_key=scenario.key(team_id=public_team), + team_models=team_models, + ) + for group in (*down.values(), *refusing.values(), *team_models.values()): + wait_for_worker_readiness(rig.gateway, group, 1, _PROXY_WORKERS) + yield built + + +@_FALLBACK_CHECKED +def test_a_request_fallback_to_a_group_listing_tools_answers_a_bad_tool_entry( + rig: _Rig, reroute: _Reroute, checked: _Checked +) -> None: + marker: Final = _marker() + response: Final = checked.send( + rig, + reroute.down[checked.provider], + marker, + {"tools": [checked.tool, checked.bad_entry], "fallbacks": [reroute.lists_tools[checked.provider]]}, + ) + _assert_failed_over(rig, checked, response, marker, ()) + + +def test_a_generic_request_fallback_to_a_group_listing_tools_answers_a_bad_tool_entry( + rig: _Rig, reroute: _Reroute +) -> None: + marker: Final = _marker() + checked: Final = _CHECKED["chat"] + response: Final = checked.send( + rig, + reroute.down["openai"], + marker, + {"tools": [checked.tool, checked.bad_entry], "fallbacks": [{"*": [rig.field_group]}]}, + ) + _assert_failed_over(rig, checked, response, marker, ()) + + +@pytest.mark.parametrize( + "fallbacks", + [[], None, lambda rig: [{rig.drop_group: [rig.field_group]}]], + ids=["empty_list", "null", "chain_of_another_group"], +) +def test_a_request_fallback_that_cannot_apply_still_rejects_a_bad_tool_entry( + rig: _Rig, fallbacks: JsonValue | Callable[[_Rig], JsonValue] +) -> None: + marker: Final = _marker() + chains: Final = fallbacks(rig) if callable(fallbacks) else fallbacks + response: Final = _send(rig, CHAT, marker, False, tools=[CHAT.tool, 5], fallbacks=chains) + _assert_held_back(rig, response, marker) + + +@pytest.mark.parametrize("owner", ["key", "team"]) +def test_router_settings_fallbacks_of_a_key_or_team_answer_a_bad_tool_entry( + rig: _Rig, reroute: _Reroute, owner: str +) -> None: + marker: Final = _marker() + response: Final = _send( + rig, + CHAT, + marker, + False, + model=reroute.down["openai"], + key=reroute.fallback_keys[owner], + tools=[CHAT.tool, _CHAT_BAD_ENTRY], + ) + _assert_failed_over(rig, _CHECKED["chat"], response, marker, ()) + + +@pytest.mark.parametrize("chain", tuple(_REFUSALS)) +def test_an_error_specific_fallback_never_sees_a_bad_tool_entry_the_first_deployment_rejects( + rig: _Rig, reroute: _Reroute, chain: str +) -> None: + marker: Final = _marker() + refusing: Final = reroute.refusing[chain] + response: Final = _send( + rig, + CHAT, + marker, + False, + model=refusing, + tools=[CHAT.tool, 5], + **{chain: [{refusing: [rig.field_group]}]}, + ) + _assert_held_back(rig, response, marker) + + +def test_a_comma_batch_lets_each_group_decide_on_a_bad_tool_entry(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _send( + rig, CHAT, marker, False, model=f"{rig.field_group},{rig.openai}", tools=[CHAT.tool, _CHAT_BAD_ENTRY] + ) + assert response.status_code == 200, response.text + assert _answer(_tag(marker)) in response.text, response.text + delivered: Final = sorted( + (_JSON_OBJECT.validate_json(request.body).get("tools") for request in _delivered(rig, marker)), key=bool + ) + assert delivered == [None] + + +def test_a_team_public_model_name_answers_a_bad_tool_entry_its_deployment_list_drops( + rig: _Rig, reroute: _Reroute +) -> None: + marker: Final = _marker() + response: Final = _send( + rig, + CHAT, + marker, + False, + model=reroute.team_models["openai"], + key=reroute.team_key, + tools=[CHAT.tool, _CHAT_BAD_ENTRY], + ) + _assert_forwarded(rig, _CHECKED["chat"], response, marker, ()) + + +def test_count_tokens_by_a_team_public_model_name_rejects_a_bad_tool_entry(rig: _Rig, reroute: _Reroute) -> None: + marker: Final = _marker() + response: Final = rig.gateway.request( + "POST", + "/v1/messages/count_tokens", + { + "model": reroute.team_models["anthropic"], + "messages": [{"role": "user", "content": marker}], + "tools": [MESSAGES.tool, 5], + }, + key=reroute.team_key, + ) + _assert_held_back(rig, response, marker) + + +def test_a_request_list_naming_web_search_options_on_a_fallback_chain_still_rejects_a_bad_tool_entry( + rig: _Rig, +) -> None: + marker: Final = _marker() + response: Final = _send( + rig, + CHAT, + marker, + False, + tools=[CHAT.tool, 5], + web_search_options=5, + additional_drop_params=["web_search_options"], + fallbacks=[{rig.openai: [rig.field_group]}], + ) + _assert_held_back(rig, response, marker) + + +def test_request_drop_params_on_a_fallback_chain_keeps_the_good_entries(rig: _Rig) -> None: + marker: Final = _marker() + response: Final = _send( + rig, + CHAT, + marker, + False, + tools=[CHAT.tool, _CHAT_BAD_ENTRY], + drop_params=True, + fallbacks=[{rig.openai: [rig.field_group]}], + ) + _assert_forwarded(rig, _CHECKED["chat"], response, marker, ("get_weather",)) + + +@_ENTRY_SHAPES +def test_a_request_list_with_a_non_string_entry_drops_tools_it_names_on_messages( + rig: _Rig, bad_entry: _BadEntry +) -> None: + marker: Final = _marker() + checked: Final = _CHECKED["messages"] + response: Final = checked.send( + rig, + rig.anthropic, + marker, + {"tools": [checked.tool, bad_entry(checked)], "additional_drop_params": ["tools", 5]}, + ) + _assert_forwarded(rig, checked, response, marker, ()) + + +@_ENTRY_SHAPES +def test_a_deployment_list_with_a_non_string_entry_drops_tools_it_names_on_messages( + rig: _Rig, bad_entry: _BadEntry +) -> None: + marker: Final = _marker() + checked: Final = _CHECKED["messages"] + with rig.gateway.scenario() as scenario: + group: Final = scenario.model(**_PROVIDERS["anthropic"](rig.wire.url), additional_drop_params=["tools", 5]) + wait_for_worker_readiness(rig.gateway, group, 1, _PROXY_WORKERS) + response: Final = checked.send(rig, group, marker, {"tools": [checked.tool, bad_entry(checked)]}) + _assert_forwarded(rig, checked, response, marker, ()) + + +@_TOKEN_COUNTS +def test_token_counting_with_a_request_list_with_a_non_string_entry_drops_the_bad_entry( + rig: _Rig, checked: _Checked +) -> None: + marker: Final = _marker() + response: Final = checked.send( + rig, rig.anthropic, marker, {"tools": [checked.tool, 5], "additional_drop_params": ["tools", 5]} + ) + _assert_forwarded(rig, checked, response, marker, ("get_weather",)) + + +def _alias(suffix: str, variant: str) -> str: + return f"malformed-tools-alias-{variant}-{suffix}" + + +def _routing_group(suffix: str, provider: str) -> str: + return f"malformed-tools-routed-{provider}-{suffix}" + + +def _wildcard(suffix: str) -> str: + return f"malformed-tools-wild-{suffix}" + + +def _direct(suffix: str) -> str: + return f"malformed-tools-direct-{suffix}" + + +def _down_models(wire: str, suffix: str) -> tuple[dict[str, JsonValue], ...]: + return tuple( + {"model_name": _owned_name(suffix, "down", provider), "litellm_params": _PROVIDERS[provider](f"{wire}/down")} + for provider in ("openai", "anthropic") + ) + + +def _rerouting_models(wire: str, suffix: str) -> tuple[dict[str, JsonValue], ...]: + lists_tools: Final = _owned_name(suffix, "lists_tools", "openai") + return ( + *_down_models(wire, suffix), + { + "model_name": _owned_name(suffix, "auto", "openai"), + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_default_model": lists_tools, + "complexity_router_config": { + "tiers": dict.fromkeys(("SIMPLE", "MEDIUM", "COMPLEX", "REASONING"), lists_tools) + }, + }, + }, + { + "model_name": _owned_name(suffix, "direct", "openai"), + "litellm_params": { + **_PROVIDERS["openai"](wire), + "model": f"openai/{_direct(suffix)}", + "additional_drop_params": ["tools"], + }, + }, + { + "model_name": f"{_wildcard(suffix)}/*", + "litellm_params": {**_PROVIDERS["openai"](wire), "model": "openai/*", "additional_drop_params": ["tools"]}, + }, + ) + + +def _rerouting_settings(suffix: str) -> dict[str, JsonValue]: + return { + "fallbacks": [ + {_owned_name(suffix, "down", provider): [_owned_name(suffix, "lists_tools", provider)]} + for provider in ("openai", "anthropic") + ], + "model_group_alias": { + _alias(suffix, variant): _owned_name(suffix, variant, "openai") for variant in ("lists_tools", "plain") + }, + "routing_groups": [ + { + "group_name": _routing_group(suffix, provider), + "models": [_owned_name(suffix, "lists_tools", provider)], + "routing_strategy": "simple-shuffle", + } + for provider in ("openai", "anthropic") + ], + "num_retries": 0, + "disable_cooldowns": True, + } + + +@pytest.fixture(scope="module") +def rerouting(rig: _Rig, tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Owned]: + directory: Final = tmp_path_factory.mktemp("malformed-tools-rerouting") + setup: Final = _OwnedSetup(router_settings=_rerouting_settings, extra_models=_rerouting_models) + with _owned_proxy(rig, directory, setup) as owned: + yield owned + + +@pytest.fixture(scope="module") +def default_fallbacks(rig: _Rig, tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Owned]: + directory: Final = tmp_path_factory.mktemp("malformed-tools-default-fallbacks") + setup: Final = _OwnedSetup( + router_settings=lambda suffix: { + "default_fallbacks": [_owned_name(suffix, "lists_tools", "openai")], + "num_retries": 0, + "disable_cooldowns": True, + }, + extra_models=_down_models, + ) + with _owned_proxy(rig, directory, setup) as owned: + yield owned + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@pytest.mark.parametrize("checked", (_CHECKED["chat"], _CHECKED["messages"]), ids=("chat", "messages")) +def test_a_router_fallback_to_a_group_listing_tools_answers_a_bad_tool_entry( + rerouting: _Owned, checked: _Checked +) -> None: + marker: Final = _marker() + model: Final = _owned_name(rerouting.suffix, "down", checked.provider) + response: Final = checked.send(rerouting.rig, model, marker, {"tools": [checked.tool, checked.bad_entry]}) + _assert_failed_over(rerouting.rig, checked, response, marker, ()) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@pytest.mark.parametrize( + ("variant", "fields"), + [("plain", {}), ("down", {"fallbacks": []})], + ids=["group_outside_the_chain", "chain_cleared_by_the_request"], +) +def test_a_router_fallback_that_cannot_apply_still_rejects_a_bad_tool_entry( + rerouting: _Owned, variant: str, fields: dict[str, JsonValue] +) -> None: + marker: Final = _marker() + model: Final = _owned_name(rerouting.suffix, variant, "openai") + response: Final = _send(rerouting.rig, CHAT, marker, False, model=model, tools=[CHAT.tool, 5], **fields) + _assert_held_back(rerouting.rig, response, marker) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@_TOKEN_COUNTS +def test_token_counting_on_a_group_with_a_router_fallback_rejects_a_bad_tool_entry( + rerouting: _Owned, checked: _Checked +) -> None: + marker: Final = _marker() + model: Final = _owned_name(rerouting.suffix, "down", "anthropic") + response: Final = checked.send(rerouting.rig, model, marker, {"tools": [checked.tool, 5]}) + _assert_held_back(rerouting.rig, response, marker) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_an_alias_of_a_group_listing_tools_answers_a_bad_tool_entry(rerouting: _Owned) -> None: + marker: Final = _marker() + response: Final = _send( + rerouting.rig, + CHAT, + marker, + False, + model=_alias(rerouting.suffix, "lists_tools"), + tools=[CHAT.tool, _CHAT_BAD_ENTRY], + ) + _assert_forwarded(rerouting.rig, _CHECKED["chat"], response, marker, ()) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_an_alias_of_a_plain_group_rejects_a_bad_tool_entry(rerouting: _Owned) -> None: + marker: Final = _marker() + response: Final = _send( + rerouting.rig, CHAT, marker, False, model=_alias(rerouting.suffix, "plain"), tools=[CHAT.tool, 5] + ) + _assert_held_back(rerouting.rig, response, marker) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_a_routing_group_over_a_group_listing_tools_answers_a_bad_tool_entry(rerouting: _Owned) -> None: + marker: Final = _marker() + response: Final = _send( + rerouting.rig, + CHAT, + marker, + False, + model=_routing_group(rerouting.suffix, "openai"), + tools=[CHAT.tool, _CHAT_BAD_ENTRY], + ) + _assert_forwarded(rerouting.rig, _CHECKED["chat"], response, marker, ()) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_count_tokens_by_a_routing_group_rejects_a_bad_tool_entry(rerouting: _Owned) -> None: + marker: Final = _marker() + response: Final = _count_tokens( + rerouting.rig, _routing_group(rerouting.suffix, "anthropic"), marker, {"tools": [MESSAGES.tool, 5]} + ) + _assert_held_back(rerouting.rig, response, marker) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@pytest.mark.parametrize( + ("model", "fields"), + [ + (lambda suffix: _owned_name(suffix, "auto", "openai"), {}), + (lambda suffix: f"{_wildcard(suffix)}/{_OPENAI_BACKEND}", {}), + (lambda suffix: f"openai/{_direct(suffix)}", {}), + (lambda suffix: f"openai/{_direct(suffix)}", {"specific_deployment": True}), + ], + ids=["auto_router", "wildcard_route", "model_string", "specific_deployment"], +) +def test_a_model_the_router_resolves_later_answers_a_bad_tool_entry_its_deployment_list_drops( + rerouting: _Owned, model: Callable[[str], str], fields: dict[str, JsonValue] +) -> None: + marker: Final = _marker() + response: Final = _send( + rerouting.rig, + CHAT, + marker, + False, + model=model(rerouting.suffix), + tools=[CHAT.tool, _CHAT_BAD_ENTRY], + **fields, + ) + _assert_forwarded(rerouting.rig, _CHECKED["chat"], response, marker, ()) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_default_fallbacks_to_a_group_listing_tools_answer_a_bad_tool_entry(default_fallbacks: _Owned) -> None: + marker: Final = _marker() + response: Final = _send( + default_fallbacks.rig, + CHAT, + marker, + False, + model=_owned_name(default_fallbacks.suffix, "down", "openai"), + tools=[CHAT.tool, _CHAT_BAD_ENTRY], + ) + _assert_failed_over(default_fallbacks.rig, _CHECKED["chat"], response, marker, ()) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_a_user_config_model_named_like_a_proxy_group_drops_tools_its_own_list_names(client_side: _Owned) -> None: + marker: Final = _marker() + name: Final = _owned_name(client_side.suffix, "plain", "openai") + user_config: Final[dict[str, JsonValue]] = { + "model_list": [ + { + "model_name": name, + "litellm_params": { + **_PROVIDERS["openai"](client_side.rig.wire.url), + "additional_drop_params": ["tools"], + }, + } + ], + } + response: Final = _send( + client_side.rig, + CHAT, + marker, + False, + model=name, + tools=[CHAT.tool, _CHAT_BAD_ENTRY], + user_config=user_config, + ) + _assert_forwarded(client_side.rig, _CHECKED["chat"], response, marker, ()) + + +_GUARDED_NAMES: Final = ("chat", "messages", "responses", "generate_content") +_EACH_GUARDED: Final = pytest.mark.parametrize( + "checked", tuple(_CHECKED[name] for name in _GUARDED_NAMES), ids=_GUARDED_NAMES +) +_GUARDRAIL_VENDOR_PATH: Final = "/beta/litellm_basic_guardrail_api" + + +@dataclass(frozen=True, slots=True) +class _Guarded: + rig: _Rig + vendor: Wire + models: Mapping[str, str] + + +def _guarded_config(path: Path, wire: str, vendor: str, models: Mapping[str, str]) -> Path: + path.write_text( + json.dumps( + { + "model_list": [ + {"model_name": name, "litellm_params": _PROVIDERS[provider](wire)} + for provider, name in models.items() + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + }, + "guardrails": [ + { + "guardrail_name": "malformed-tools-vendor", + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": True, + "api_base": vendor, + "api_key": "synthetic-guardrail-key", + }, + } + ], + } + ) + ) + return path + + +def _guardrail_verdict(request: Request) -> Reply: + assert request.target == _GUARDRAIL_VENDOR_PATH, request.target + return Reply(body=json.dumps({"action": "NONE"}).encode()) + + +@pytest.fixture(scope="module") +def guarded(rig: _Rig, tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Guarded]: + directory: Final = tmp_path_factory.mktemp("malformed-tools-guarded") + suffix: Final = uuid.uuid4().hex[:12] + models: Final = MappingProxyType( + {provider: f"malformed-tools-guarded-{provider}-{suffix}" for provider in _PROVIDERS} + ) + with wire_server(_guardrail_verdict) as vendor: + config: Final = _guarded_config(directory / "guarded.yaml", rig.wire.url, vendor.url, models) + with owned_proxy_process(rig.gateway, directory, {}, config=config, workers=2) as proxy: + yield _Guarded(replace(rig, gateway=proxy.gateway), vendor, models) + + +def _vendor_payloads(guarded: _Guarded, marker: str) -> tuple[dict[str, JsonValue], ...]: + return tuple( + _JSON_OBJECT.validate_json(request.body) + for request in guarded.vendor.drain() + if marker in request.body.decode() + ) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +@_EACH_GUARDED +def test_a_pre_call_guardrail_passes_well_formed_tools_through_to_the_provider( + guarded: _Guarded, checked: _Checked +) -> None: + marker: Final = _marker() + response: Final = checked.send(guarded.rig, guarded.models[checked.provider], marker, {"tools": [checked.tool]}) + _assert_forwarded(guarded.rig, checked, response, marker, ("get_weather",)) + payloads: Final = _vendor_payloads(guarded, marker) + assert len(payloads) == 1, payloads + assert all(payload.get("tools") is None or isinstance(payload["tools"], list) for payload in payloads), payloads + + +_PROVIDER_WIRES: Final[Mapping[str, tuple[_Family, Callable[[bool], str]]]] = MappingProxyType( + { + "openai": (CHAT, lambda stream: "/v1/chat/completions"), + "openai_responses": (RESPONSES, lambda stream: "/v1/responses"), + "anthropic": (MESSAGES, lambda stream: "/v1/messages"), + "gemini": (GENERATE, lambda stream: ":streamGenerateContent" if stream else ":generateContent"), + } +) +_NATIVE_PROVIDERS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( + { + "chat": ("openai",), + "messages": ("anthropic",), + "responses": ("openai", "openai_responses"), + "generate_content": ("gemini",), + } +) +_ROUTED_ELSEWHERE: Final[Mapping[tuple[str, str], str]] = MappingProxyType({("messages", "openai"): "openai_responses"}) +_BRIDGES: Final = tuple( + (family, provider) + for family, provider in product(_FAMILIES, _PROVIDER_WIRES) + if provider not in _NATIVE_PROVIDERS[family.name] +) + + +@pytest.fixture(scope="module") +def bridge_groups(rig: _Rig) -> Iterator[Mapping[str, str]]: + with rig.gateway.scenario() as scenario: + responses_group: Final = scenario.model( + model=f"openai/responses/{_OPENAI_BACKEND}", api_base=f"{rig.wire.url}/v1", api_key=_PROVIDER_KEY + ) + wait_for_worker_readiness(rig.gateway, responses_group, 1, _PROXY_WORKERS) + yield MappingProxyType( + { + "openai": rig.openai, + "openai_responses": responses_group, + "anthropic": rig.anthropic, + "gemini": rig.gemini, + } + ) + + +@_STREAMS +@pytest.mark.parametrize( + ("family", "provider"), _BRIDGES, ids=tuple(f"{family.name}_to_{provider}" for family, provider in _BRIDGES) +) +def test_well_formed_tools_cross_the_bridge_to_another_provider( + rig: _Rig, bridge_groups: Mapping[str, str], family: _Family, provider: str, stream: bool +) -> None: + marker: Final = _marker() + response: Final = _send(rig, family, marker, stream, model=bridge_groups[provider], tools=[family.tool]) + assert response.status_code == 200, response.text + assert _answer(_tag(marker)) in response.text, response.text + named_as, target = _PROVIDER_WIRES[_ROUTED_ELSEWHERE.get((family.name, provider), provider)] + assert _tool_names(named_as, _one_delivery(rig, marker, target(stream))) == ("get_weather",) diff --git a/tests/integration/streaming/test_google_genai_stream_usage_settings_wire.py b/tests/integration/streaming/test_google_genai_stream_usage_settings_wire.py index 85017667c74..796e706c569 100644 --- a/tests/integration/streaming/test_google_genai_stream_usage_settings_wire.py +++ b/tests/integration/streaming/test_google_genai_stream_usage_settings_wire.py @@ -1,21 +1,19 @@ import json -import re import threading import uuid from collections.abc import Callable, Mapping, Sequence -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future, ThreadPoolExecutor from contextlib import ExitStack from hashlib import sha256 from pathlib import Path from typing import Final import httpx -import psutil import pytest import yaml from integration._support.client import Gateway, eventually, held, object_value from integration._support.database import read_rows -from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.process import WorkerHealthcheck, graceful_stop_seconds, owned_proxy_process from integration._support.vertex import service_account_json from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import JsonValue, TypeAdapter @@ -38,7 +36,6 @@ _GEMINI_USAGE: Final = { "totalTokenCount": _PROMPT_TOKENS + _COMPLETION_TOKENS, } _ZERO_USAGE: Final = {"promptTokenCount": 0, "candidatesTokenCount": 0, "totalTokenCount": 0} -_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") _BURST_SELECT: Final = ( 'SELECT api_key, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs"' " WHERE api_key = ANY(string_to_array(%s, ','))" @@ -67,6 +64,7 @@ _UPSTREAM_ASKS_FOR_USAGE: Final = { }, } _SETTING_VALUES: Final = {"always_off": False, "always_on": True} +_STREAMS_PER_WORKER: Final = 6 def _openai_frame(chunk: Mapping[str, JsonValue], response_id: str) -> bytes: @@ -273,20 +271,15 @@ def _stream_or_dropped(client: httpx.Client, key: str) -> httpx.Response | None: return None -def _local_port(client: httpx.Client) -> int: - with client.stream("GET", "/health/liveliness") as response: - port: Final = int(response.extensions["network_stream"].get_extra_info("client_addr")[1]) - response.read() - assert response.status_code == 200, response.text - return port - - -def _accepted_client_ports(pid: int, proxy_port: int) -> frozenset[int]: - return frozenset( - connection.raddr.port - for connection in psutil.Process(pid).net_connections(kind="tcp") - if connection.raddr and connection.laddr.port == proxy_port +def _held_streams( + stack: ExitStack, pool: ThreadPoolExecutor, url: str, keys: Sequence[str], wire: Wire, held: int +) -> tuple[Future[httpx.Response | None], ...]: + clients: Final = tuple(stack.enter_context(httpx.Client(base_url=url, timeout=60, trust_env=False)) for _ in keys) + futures: Final = tuple( + pool.submit(_stream_or_dropped, client, key) for client, key in zip(clients, keys, strict=True) ) + eventually(wire.received.qsize, lambda size: size >= held, seconds=60) + return futures @pytest.mark.timeout(2 * graceful_stop_seconds() + 120) @@ -296,35 +289,27 @@ def test_worker_killed_mid_stream_leaves_the_sibling_billing_each_stream_once(ga wire: Final = stack.enter_context(wire_server(_peer(gate))) stack.callback(gate.set) config: Final = _owned_config(tmp_path / "worker-kill.yaml", gateway, wire, {}) - owned: Final = stack.enter_context(owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2)) - workers: Final = eventually( - lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())), - lambda pids: len(pids) == 2, - seconds=30, + owned: Final = stack.enter_context( + owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2, healthcheck=WorkerHealthcheck.LENIENT) ) + victim, survivor = sorted(eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=30)) scenario: Final = stack.enter_context(owned.gateway.scenario()) - proxy_url: Final = owned.gateway.client.base_url - clients: Final = tuple( - stack.enter_context(httpx.Client(base_url=str(proxy_url), timeout=60, trust_env=False)) for _ in range(12) - ) - keys: Final = tuple(scenario.key(models=[_ADAPTER_MODEL]) for _ in clients) - ports: Final = tuple(_local_port(client) for client in clients) - pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients))) - futures: Final = tuple( - pool.submit(_stream_or_dropped, client, key) for client, key in zip(clients, keys, strict=True) - ) - eventually(wire.received.qsize, lambda size: size >= len(clients), seconds=60) - shares: Final = {pid: _accepted_client_ports(pid, proxy_url.port or 0) & frozenset(ports) for pid in workers} - assert sum(map(len, shares.values())) == len(clients), shares - victim: Final = min((pid for pid in workers if shares[pid]), key=lambda pid: len(shares[pid])) + victim_keys: Final = tuple(scenario.key(models=[_ADAPTER_MODEL]) for _ in range(_STREAMS_PER_WORKER)) + survivor_keys: Final = tuple(scenario.key(models=[_ADAPTER_MODEL]) for _ in range(_STREAMS_PER_WORKER)) + proxy_url: Final = str(owned.gateway.client.base_url) + pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=2 * _STREAMS_PER_WORKER)) + with owned.suspended(survivor): + on_victim: Final = _held_streams(stack, pool, proxy_url, victim_keys, wire, _STREAMS_PER_WORKER) + with owned.suspended(victim): + on_survivor: Final = _held_streams(stack, pool, proxy_url, survivor_keys, wire, 2 * _STREAMS_PER_WORKER) with owned.replacing_worker(victim) as kill: kill() gate.set() - results: Final = tuple(future.result(timeout=60) for future in futures) - survivors: Final = tuple( - (key, result) for port, key, result in zip(ports, keys, results, strict=True) if port not in shares[victim] - ) - assert survivors, shares + dropped: Final = tuple(future.result(timeout=60) for future in on_victim) + survivors: Final = tuple( + zip(survivor_keys, (future.result(timeout=60) for future in on_survivor), strict=True) + ) + assert dropped == (None,) * _STREAMS_PER_WORKER, dropped for _, result in survivors: assert result is not None _assert_one_stop_with(result, _GEMINI_USAGE) diff --git a/tests/unit/integration_support/test_process.py b/tests/unit/integration_support/test_process.py index 26306a2d39a..2106436edb8 100644 --- a/tests/unit/integration_support/test_process.py +++ b/tests/unit/integration_support/test_process.py @@ -379,48 +379,47 @@ def test_a_replacing_block_that_never_kills_its_worker_fails_and_leaves_it_servi assert owned.worker_pids() == frozenset((worker,)) -def _kill_of(owned: OwnedProxy, worker: int) -> WorkerKill: - return WorkerKill(psutil.Process(worker), psutil.Process(owned.process.pid), 5, threading.Event()) +@contextmanager +def _unreaped_child() -> Iterator[psutil.Process]: + with subprocess.Popen((sys.executable, "-I", "-c", "import time; time.sleep(60)")) as child: + try: + yield psutil.Process(child.pid) + finally: + child.kill() -def test_a_worker_kill_returns_once_the_supervisor_reaped_the_victim_and_cannot_fire_twice(tmp_path: Path) -> None: - with _supervised(tmp_path, WorkerHealthcheck.STRICT) as (owned, worker): - kill: Final = _kill_of(owned, worker) +def test_a_worker_kill_returns_once_the_victim_is_dead_before_anything_reaps_it_and_cannot_fire_twice() -> None: + with _unreaped_child() as victim: + kill: Final = WorkerKill(victim, threading.Event()) kill() - assert not psutil.pid_exists(worker) + assert victim.status() == psutil.STATUS_ZOMBIE assert kill.fired.is_set() with pytest.raises(AssertionError, match="already killed"): kill() -def test_a_frozen_victim_stays_stopped_until_the_kill(tmp_path: Path) -> None: - with _supervised(tmp_path, WorkerHealthcheck.STRICT) as (owned, worker): - kill: Final = _kill_of(owned, worker) +def test_a_frozen_victim_stays_stopped_until_the_kill() -> None: + with _unreaped_child() as victim: + kill: Final = WorkerKill(victim, threading.Event()) assert kill.freeze().status() == psutil.STATUS_STOPPED kill() - assert not psutil.pid_exists(worker) + assert victim.status() == psutil.STATUS_ZOMBIE -def test_a_kill_under_a_suspended_supervisor_returns_with_the_victim_left_unreaped(tmp_path: Path) -> None: - with _supervised(tmp_path, WorkerHealthcheck.LENIENT) as (owned, worker): - kill: Final = _kill_of(owned, worker) - with owned.suspended(owned.process.pid): - kill() - assert psutil.Process(worker).status() == psutil.STATUS_ZOMBIE - eventually(lambda: psutil.pid_exists(worker), lambda exists: not exists, seconds=5) +def _kill_after_freezing_the_supervisor(owned: OwnedProxy, worker: int, supervisor: psutil.Process) -> None: + with owned.replacing_worker(worker) as kill: + supervisor.suspend() + kill() -def test_a_kill_fails_when_the_supervisor_never_reaps_the_victim(tmp_path: Path) -> None: +def test_a_replacing_block_fails_when_the_supervisor_never_reaps_the_killed_worker(tmp_path: Path) -> None: with _supervised(tmp_path, WorkerHealthcheck.STRICT) as (owned, worker): unreaping: Final = psutil.Process(owned.process.pid) - kill: Final = WorkerKill(psutil.Process(worker), psutil.Process(os.getpid()), 0.5, threading.Event()) - unreaping.suspend() try: with pytest.raises(AssertionError, match="did not reap"): - kill() + _kill_after_freezing_the_supervisor(owned, worker, unreaping) finally: unreaping.resume() - assert not kill.fired.is_set() def test_a_strict_healthcheck_proxy_refuses_suspension_before_stopping_anything(tmp_path: Path) -> None: diff --git a/tests/unit/integrations/test_prometheus_deployment_state_proxy_rejects.py b/tests/unit/integrations/test_prometheus_deployment_state_proxy_rejects.py index fedfc2b0848..8eefee1dfcf 100644 --- a/tests/unit/integrations/test_prometheus_deployment_state_proxy_rejects.py +++ b/tests/unit/integrations/test_prometheus_deployment_state_proxy_rejects.py @@ -9,12 +9,14 @@ response. A failure raised after the router picked a deployment (a post-call guardrail block, a provider error) carries no flag and keeps its deployment labels. """ +from typing import Final + import pytest from fastapi import HTTPException from prometheus_client import REGISTRY from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY -from litellm.exceptions import GuardrailRaisedException +from litellm.exceptions import GuardrailRaisedException, MalformedRequestFieldError from litellm.integrations.prometheus import PrometheusLogger from litellm.proxy._types import ProxyException from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError @@ -99,3 +101,15 @@ def test_failure_after_routing_still_marks_deployment_partial_outage(failure: Ex assert _model_id_values(logger.litellm_deployment_state) == {"dep-1"} assert _model_id_values(logger.litellm_deployment_failure_responses) == {"dep-1"} + + +def test_malformed_field_rejection_after_routing_leaves_deployment_healthy() -> None: + logger: Final = PrometheusLogger() + rejection: Final = MalformedRequestFieldError( + message="tools: Input should be a valid list", model="internal-model", llm_provider="openai" + ) + + logger.set_llm_deployment_failure_metrics(_attributed_failure_kwargs(rejection, rejected_before_routing=False)) + + assert logger.litellm_deployment_state._metrics == {} + assert _model_id_values(logger.litellm_deployment_failure_responses) == {""} diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index c8dbbd00d66..7291c4858f6 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1,4 +1,6 @@ +from typing import Final + import pytest import litellm @@ -49,11 +51,12 @@ def test_web_search_cost_high(): ) -def test_get_web_search_options_preserves_explicit_none_error(): - with pytest.raises(TypeError): - StandardBuiltInToolCostTracking.get_web_search_options( - {"web_search_options": None, "tools": [{"type": "web_search_preview"}]} - ) +def test_get_web_search_options_reads_tools_when_options_are_null() -> None: + options: Final = StandardBuiltInToolCostTracking.get_web_search_options( + {"web_search_options": None, "tools": [{"type": "web_search_preview"}]} + ) + + assert options == {"type": "web_search_preview"} def test_get_web_search_options_accepts_non_list_tool_iterables(): @@ -1070,3 +1073,13 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None: usage = Usage(prompt_tokens=1840, completion_tokens=412, total_tokens=2252) assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None + + +@pytest.mark.parametrize( + "kwargs", + [{"tools": 5}, {"web_search_options": None}, {"web_search_options": 5}], + ids=["int_tools", "null_web_search_options", "int_web_search_options"], +) +def test_built_in_tool_lookups_ignore_malformed_request_values(kwargs: dict[str, object]) -> None: + assert StandardBuiltInToolCostTracking.get_web_search_options(kwargs) is None + assert StandardBuiltInToolCostTracking.get_file_search_tool_call(kwargs) is None diff --git a/tests/unit/litellm_core_utils/test_tool_fields.py b/tests/unit/litellm_core_utils/test_tool_fields.py new file mode 100644 index 00000000000..23d7de98fbc --- /dev/null +++ b/tests/unit/litellm_core_utils/test_tool_fields.py @@ -0,0 +1,187 @@ +import datetime +from collections.abc import Callable, Mapping +from typing import Final + +import pytest + +import litellm +from litellm import Router +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.utils import async_pre_call_deployment_hook + +_GOOD_TOOL: Final = {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}} +_MALFORMED_TOOLS: Final = ( + 5, + "get_weather", + ["get_weather"], + {"type": "function"}, + [{"type": "function", "function": {"name": "get_weather", "parameters": "object"}}], +) + + +def _deployment(name: str, **params: object) -> Mapping[str, object]: + return { + "model_name": name, + "litellm_params": {"model": "openai/gpt-5.2", "api_key": "sk-test", "mock_response": f"served by {name}", **params}, + } + + +def _proxy_request(model: str, route: str = "acompletion", **fields: object) -> Mapping[str, object]: + return { + "model": model, + "messages": [{"role": "user", "content": "weather in Paris?"}], + "proxy_server_request": {"url": "/v1/chat/completions", "body": {}}, + "litellm_logging_obj": Logging( + model=model, + messages=[], + stream=False, + call_type=route, + start_time=datetime.datetime.now(), + litellm_call_id="call-1", + function_id="fn-1", + ), + "num_retries": 0, + **fields, + } + + +@pytest.mark.parametrize("tools", _MALFORMED_TOOLS) +@pytest.mark.asyncio +async def test_malformed_tools_fall_back_to_a_deployment_that_drops_them(tools: object) -> None: + router: Final = Router( + model_list=[_deployment("primary"), _deployment("backup", additional_drop_params=["tools"])], + fallbacks=[{"primary": ["backup"]}], + ) + + response: Final = await router.acompletion(**_proxy_request("primary", tools=tools)) + + assert response.choices[0].message.content == "served by backup" + + +@pytest.mark.asyncio +async def test_fallback_on_unsupported_params_false_keeps_the_400() -> None: + router: Final = Router( + model_list=[_deployment("primary"), _deployment("backup", additional_drop_params=["tools"])], + fallbacks=[{"primary": ["backup"]}], + fallback_on_unsupported_params=False, + ) + + with pytest.raises(litellm.BadRequestError): + await router.acompletion(**_proxy_request("primary", tools=5)) + + +@pytest.mark.asyncio +async def test_a_rejected_request_cools_down_no_deployment() -> None: + cools_on_any_400: Final = {"allowed_fails_policy": {"BadRequestErrorAllowedFails": 0}, "cooldown_time": 120} + router: Final = Router( + model_list=[ + {**_deployment("primary"), "model_info": {"id": "primary", **cools_on_any_400}}, + {**_deployment("backup"), "model_info": {"id": "backup", **cools_on_any_400}}, + ], + fallbacks=[{"primary": ["backup"]}], + ) + + with pytest.raises(litellm.BadRequestError): + await router.acompletion(**_proxy_request("primary", tools=5)) + + primary: Final = await router.acompletion(**_proxy_request("primary")) + backup: Final = await router.acompletion(**_proxy_request("backup")) + assert (primary.choices[0].message.content, backup.choices[0].message.content) == ( + "served by primary", + "served by backup", + ) + + +@pytest.mark.parametrize("tools", _MALFORMED_TOOLS) +@pytest.mark.parametrize( + "deployment", + [_deployment("keeps"), _deployment("keeps", tools=[_GOOD_TOOL])], + ids=["plain", "with_configured_tools"], +) +@pytest.mark.asyncio +async def test_malformed_tools_answer_400_when_the_served_deployment_keeps_tools( + tools: object, deployment: Mapping[str, object] +) -> None: + router: Final = Router(model_list=[deployment]) + + with pytest.raises(litellm.BadRequestError) as raised: + await router.acompletion(**_proxy_request("keeps", tools=tools)) + + assert (raised.value.status_code, raised.value.param, raised.value.type) == ( + 400, + "tools", + "invalid_request_error", + ) + + +@pytest.mark.asyncio +async def test_a_request_drop_params_false_overrides_a_deployment_that_drops() -> None: + router: Final = Router(model_list=[_deployment("drops", drop_params=True)]) + + with pytest.raises(litellm.BadRequestError): + await router.acompletion(**_proxy_request("drops", tools=5, drop_params=False)) + + +@pytest.mark.parametrize( + "route", ["acompletion", "aresponses", "anthropic_messages", "agenerate_content", "acreate_interaction"] +) +@pytest.mark.asyncio +async def test_drop_params_drops_every_malformed_tool_field(route: str) -> None: + request: Final = _proxy_request( + "drops", route, drop_params=True, tools=5, tool_choice="auto", web_search_options="high" + ) + + sent: Final = await async_pre_call_deployment_hook(dict(request), route) + + assert {"tools", "tool_choice", "web_search_options"}.isdisjoint(sent) + assert sent["messages"] == request["messages"] + + +@pytest.mark.asyncio +async def test_additional_drop_params_drops_only_the_malformed_tool_entries() -> None: + request: Final = _proxy_request("drops", additional_drop_params=["tools"], tools=[_GOOD_TOOL, "get_weather"]) + + sent: Final = await async_pre_call_deployment_hook(dict(request), "acompletion") + + assert sent["tools"] == [_GOOD_TOOL] + + +@pytest.mark.asyncio +async def test_an_sdk_call_keeps_the_tools_it_passed() -> None: + tools: Final = (_GOOD_TOOL,) + request: Final = { + key: value for key, value in _proxy_request("sdk", tools=tools).items() if key != "proxy_server_request" + } + + sent: Final = await async_pre_call_deployment_hook(dict(request), "acompletion") + + assert sent["tools"] is tools + + +@pytest.mark.parametrize("schema", ["", [], False, 0, {}, '{"type": "object"}']) +@pytest.mark.parametrize( + "tool", + [ + lambda schema: {"type": "function", "function": {"name": "get_weather", "parameters": schema}}, + lambda schema: {"type": "function", "name": "get_weather", "parameters": schema}, + lambda schema: {"name": "get_weather", "input_schema": schema}, + ], + ids=["chat", "responses", "messages"], +) +@pytest.mark.asyncio +async def test_empty_and_json_string_schemas_reach_the_deployment( + tool: Callable[[object], Mapping[str, object]], schema: object +) -> None: + tools: Final = [tool(schema)] + + sent: Final = await async_pre_call_deployment_hook(dict(_proxy_request("keeps", tools=tools)), "acompletion") + + assert sent["tools"] is tools + + +@pytest.mark.parametrize("tools", [(_GOOD_TOOL,), [{"type": "web_search", "function": None}]], ids=["tuple", "null"]) +@pytest.mark.asyncio +async def test_tools_core_accepts_reach_the_deployment(tools: object) -> None: + sent: Final = await async_pre_call_deployment_hook(dict(_proxy_request("keeps", tools=tools)), "acompletion") + + assert sent["tools"] is tools diff --git a/tests/unit/proxy/_experimental/mcp_server/test_subscriptions.py b/tests/unit/proxy/_experimental/mcp_server/test_subscriptions.py index a5d42e58b99..a97ab04c506 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_subscriptions.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_subscriptions.py @@ -1,4 +1,5 @@ import asyncio +from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, Mock @@ -19,6 +20,21 @@ from litellm.types.mcp import MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer +def allow_one_request(monkeypatch: pytest.MonkeyPatch, user_id: str, server_id: str) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + windows: Final = {(user_id, server_id): asyncio.Semaphore(1)} + + async def enforce(user: UserAPIKeyAuth | None, server: MCPServer) -> None: + window: Final = windows[(user.user_id if user is not None else None, server.server_id)] + if window.locked(): + raise ProxyRateLimitError(detail=f"{server_id} RPM exceeded") + await window.acquire() + + monkeypatch.setattr(proxy_server, "proxy_logging_obj", SimpleNamespace(enforce_mcp_server_rate_limits=enforce)) + + def context(session: ServerSession, revision: str = "2026-07-28") -> ServerRequestContext: return ServerRequestContext( session=session, @@ -356,3 +372,117 @@ async def test_proxy_route_rejects_legacy_resource_subscription_before_setup(met await getattr(server, method)(context(Mock(spec=ServerSession)), params) finally: server._mcp_proxy_mode.reset(token) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("client_requests", "tools_list_admitted"), + [((), True), (("subscriptions/listen",), False), (("resources/subscribe",), False)], +) +async def test_only_client_initiated_listeners_consume_server_rate_limit( + monkeypatch: pytest.MonkeyPatch, client_requests: tuple[str, ...], tools_list_admitted: bool +) -> None: + from litellm.proxy._experimental.mcp_server import operations + + allow_one_request(monkeypatch, "alice", "upstream") + opened: Final = asyncio.Queue[None]() + + async def hold_upstream(*_: object, **__: object) -> None: + opened.put_nowait(None) + await asyncio.Event().wait() + + client: Final = Mock(spec=MCPClient) + client.run_with_session = AsyncMock(side_effect=hold_upstream) + server: Final = MCPServer(server_id="upstream", name="upstream", url="http://localhost/mcp", transport="http") + target: Final = subscriptions.SubscriptionTarget(server, client, "synthetic-fingerprint") + + async def resolve(_: OperationContext) -> tuple[subscriptions.SubscriptionTarget, ...]: + return (target,) + + service: Final = subscriptions.GatewaySubscriptions(resolve_targets=resolve) + caller: Final = OperationContext(_caller=UserAPIKeyAuth(user_id="alice")) + authorize: Final = AsyncMock(return_value=caller) + session: Final = Mock(spec=ServerSession) + session.client_params = Mock() + legacy: Final = context(session, "2025-11-25") + requests: Final = { + "subscriptions/listen": lambda: service.listen( + context(Mock(spec=ServerSession)), + SubscriptionsListenRequestParams(notifications=SubscriptionFilter(tools_list_changed=True)), + authorize, + "alice", + ), + "resources/subscribe": lambda: service.legacy_resource(legacy, "test://value", subscribe=True), + } + authorization: Final = subscriptions.SubscriptionAuthorization(caller, authorize, lambda *_: "alice") + starts: Final = (lambda: service.legacy_session(legacy, authorization), *(requests[r] for r in client_requests)) + tasks: Final = tuple(asyncio.create_task(start()) for start in starts) + try: + for _ in tasks: + await asyncio.wait_for(opened.get(), 1) + rejection: Final = await operations._mcp_server_rate_limit_rejection(server, caller.user_api_key_auth) + assert (rejection is None) is tools_list_admitted + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_legacy_listener_catalog_refreshes_do_not_consume_server_rate_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from mcp.types import ListToolsResult, ServerCapabilities, Tool, ToolsCapability + + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import catalog, operations + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListOk + + server: Final = MCPServer(server_id="list-changed", name="list-changed", transport=MCPTransport.http) + caller: Final = UserAPIKeyAuth(api_key="list-changed-key", user_id="alice") + allow_one_request(monkeypatch, "alice", "list-changed") + listed: Final = asyncio.Queue[None]() + + async def fetch_tools(_: MCPServer, **__: object) -> tuple[ListToolsResult, ServerListOk]: + listed.put_nowait(None) + return ListToolsResult(tools=[Tool(name="add", inputSchema={"type": "object"})]), ServerListOk(tool_count=1) + + monkeypatch.setenv("LITELLM_SALT_KEY", "list-changed-test-key") + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(operations.global_mcp_server_manager, "registry", {server.server_id: server}) + monkeypatch.setattr(operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server])) + monkeypatch.setattr(catalog, "get_filtered_server_tools", fetch_tools) + + upstream: Final = Mock() + upstream.server_capabilities = ServerCapabilities(tools=ToolsCapability(list_changed=True)) + upstream.protocol_version = "2025-11-25" + upstream.send_ping = AsyncMock() + + async def run_with_session(connected, **_: object) -> None: + await connected(upstream) + + client: Final = Mock(spec=MCPClient) + client.run_with_session = AsyncMock(side_effect=run_with_session) + target: Final = subscriptions.SubscriptionTarget(server, client, "synthetic-fingerprint") + + async def resolve(_: OperationContext) -> tuple[subscriptions.SubscriptionTarget, ...]: + return (target,) + + initial: Final = operations.prepare_context(user_api_key_auth=caller, mcp_servers=[server.server_id]) + authorize: Final = AsyncMock(return_value=initial) + session: Final = Mock(spec=ServerSession) + session.client_params = Mock() + service: Final = subscriptions.GatewaySubscriptions(resolve_targets=resolve) + listener: Final = asyncio.create_task( + service.legacy_session( + context(session, "2025-11-25"), + subscriptions.SubscriptionAuthorization(initial, authorize, lambda *_: "alice"), + ) + ) + try: + for _ in range(2): + await asyncio.wait_for(listed.get(), 5) + assert await operations._mcp_server_rate_limit_rejection(server, caller) is None + finally: + listener.cancel() + await asyncio.gather(listener, return_exceptions=True) diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index e4129ebbc39..c7bb05e5d30 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -5,13 +5,16 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objec import json import logging import unittest +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient +from litellm import Router from litellm._logging import verbose_proxy_logger from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.types.utils import TokenCountResponse class TestAnthropicEndpoints(unittest.TestCase): @@ -467,6 +470,151 @@ class TestErrorLogCarriesCallId: assert call_id in record.getMessage() +def _claude_router() -> Router: + return Router( + model_list=[ + { + "model_name": "claude-sonnet", + "litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_key": "sk-test"}, + } + ] + ) + + +_MALFORMED_TOOL_ENTRY: Final = [{"name": "get_weather", "input_schema": "object"}] +_SEVEN_TOKENS: Final = TokenCountResponse( + total_tokens=7, request_model="claude-sonnet", model_used="claude-sonnet", tokenizer_type="test" +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "tools"), + [ + ("claude-sonnet", 5), + ("claude-sonnet", ""), + ("claude-sonnet", ["get_weather"]), + ("claude-sonnet", _MALFORMED_TOOL_ENTRY), + ("team-public-name", 5), + ], + ids=["int", "empty_string", "list_of_strings", "input_schema_string", "unnamed_model_int"], +) +async def test_count_tokens_answers_400_for_malformed_tools_without_counting(model: str, tools: object) -> None: + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + request: Final = MagicMock() + request.headers = {} + counter: Final = AsyncMock() + body: Final = {"model": model, "messages": [{"role": "user", "content": "hi"}], "tools": tools} + + with ( + patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam + patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts it is never reached + patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route + ): + response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + + assert response.status_code == 400 + assert json.loads(response.body)["error"]["type"] == "invalid_request_error" + counter.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_count_tokens_counts_malformed_tool_entries_for_a_model_the_router_cannot_name() -> None: + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + request: Final = MagicMock() + request.headers = {} + counter: Final = AsyncMock(return_value=_SEVEN_TOKENS) + body: Final = {"model": "team-public-name", "messages": [{"role": "user", "content": "hi"}], "tools": _MALFORMED_TOOL_ENTRY} + + with ( + patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam + patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives + patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route + ): + response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + + assert response == {"input_tokens": 7} + assert counter.await_args.kwargs["request"].tools == _MALFORMED_TOOL_ENTRY + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("dropping", "headers"), + [ + ({"drop_params": True}, {}), + ({"additional_drop_params": ["tools"]}, {}), + ({}, {"user-agent": "claude-cli/2.1.0 (external, cli)"}), + ], + ids=["drop_params", "additional_drop_params", "claude_code_user_agent"], +) +async def test_count_tokens_drops_malformed_tools_when_drop_params_is_set( + dropping: dict[str, object], headers: dict[str, str] +) -> None: + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + request: Final = MagicMock() + request.headers = headers + counter: Final = AsyncMock(return_value=_SEVEN_TOKENS) + messages: Final = [{"role": "user", "content": "hi"}] + body: Final = {"model": "claude-sonnet", "messages": messages, "tools": 5, **dropping} + + with ( + patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam + patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives + ): + response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + + assert response == {"input_tokens": 7} + counted: Final = counter.await_args.kwargs["request"] + assert counted.tools is None + assert counted.messages == messages + + +@pytest.mark.asyncio +async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_even_with_fallbacks() -> None: + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm import Router + from litellm.proxy._types import UserAPIKeyAuth + + router: Final = Router( + model_list=[ + { + "model_name": "claude-sonnet", + "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", + "api_key": "sk-test", + "additional_drop_params": ["tools"], + }, + }, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "sk-test"}}, + ], + fallbacks=[{"claude-sonnet": ["backup"]}], + ) + request: Final = MagicMock() + request.headers = {} + counter: Final = AsyncMock(return_value=_SEVEN_TOKENS) + body: Final = {"model": "claude-sonnet", "messages": [{"role": "user", "content": "hi"}], "tools": 5} + + with ( + patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam + patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives + patch.object(proxy_server, "llm_router", new=router), # test-quality-ok: module global imported at call time; no injection seam on this route + ): + response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + + assert response == {"input_tokens": 7} + assert counter.await_args.kwargs["request"].tools is None + + class TestEventLoggingBatchEndpoint: """Test the stubbed event logging batch endpoint""" diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py index 19eb3c71a5b..f50da1bff64 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py @@ -5,6 +5,8 @@ This test file tests the Generic Guardrail API implementation, specifically focusing on metadata extraction and passing. """ +import json +import logging import os from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -16,6 +18,7 @@ import litellm from litellm import ModelResponse from litellm._version import version as litellm_version from litellm.exceptions import GuardrailRaisedException, Timeout +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, MaskedHTTPStatusError from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( GenericGuardrailAPI, @@ -38,6 +41,38 @@ def generic_guardrail(): ) +@pytest.fixture +def raw_rows_guardrail(): + return GenericGuardrailAPI( + api_base="https://api.test.guardrail.com", + headers={"Authorization": "Bearer test-key"}, + guardrail_name="test-generic-guardrail-raw-rows", + event_hook="pre_call", + default_on=True, + forward_messages_as_sent=True, + ) + + +async def _structured_messages_sent(guardrail: GenericGuardrailAPI, rows: list[dict]) -> list: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: records every body the guardrail endpoint received + + async def answer(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.content) + return httpx.Response(200, json={"action": "NONE"}, request=request) + + endpoint: Final = AsyncHTTPHandler(transport=httpx.MockTransport(answer)) + try: + with patch.object(guardrail, "async_handler", endpoint): + await guardrail.apply_guardrail( + inputs={"texts": ["Look up 123-45-6789 for me."], "structured_messages": rows}, + request_data={}, + input_type="request", + ) + finally: + await endpoint.close() + return [json.loads(body)["structured_messages"] for body in sent_bodies] + + @pytest.fixture def mock_user_api_key_dict(): """Create a mock UserAPIKeyAuth object""" @@ -638,6 +673,157 @@ class TestStructuredMessagesInResponse: assert returned_rows[1] is tool_call_row assert returned_rows[2] == {"role": "tool", "tool_call_id": "call_1", "content": '{"ssn": "[REDACTED]"}'} + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("row", "openai_row"), + [ + ( + {"role": "user", "content": "Look up 123-45-6789 for me.", "name": "pat"}, + {"role": "user", "content": "Look up 123-45-6789 for me."}, + ), + ( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": "{}"}, + "index": 0, + } + ], + }, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}} + ], + }, + ), + ( + { + "role": "user", + "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAA="}} + ], + }, + {"role": "user", "content": []}, + ), + ], + ids=["extra_key", "tool_call_index", "image_block"], + ) + async def test_rows_reach_the_guardrail_as_openai_chat_rows_by_default( + self, generic_guardrail: GenericGuardrailAPI, row, openai_row + ): + assert await _structured_messages_sent(generic_guardrail, [row]) == [[openai_row]] + + @pytest.mark.asyncio + async def test_a_returned_row_that_differs_from_the_shown_row_is_written_back_by_default( + self, generic_guardrail, mock_request_data_input + ): + sent_row = {"role": "user", "content": "my ssn is 123-45-6789", "name": "pat"} + + def answer_with_the_row_as_the_client_sent_it(url, json, headers, **_kwargs): + answer = MagicMock() + answer.json.return_value = { + "action": "GUARDRAIL_INTERVENED", + "texts": ["my ssn is [SSN]"], + "structured_messages": [sent_row], + } + answer.raise_for_status = MagicMock() + return answer + + with patch.object( + generic_guardrail.async_handler, "post", side_effect=answer_with_the_row_as_the_client_sent_it + ): + guardrailed_inputs = await generic_guardrail.apply_guardrail( + inputs={"texts": ["my ssn is 123-45-6789"], "structured_messages": [sent_row]}, + request_data=mock_request_data_input, + input_type="request", + ) + + assert guardrailed_inputs["structured_messages"] == [sent_row] + + @pytest.mark.asyncio + async def test_rows_echoed_back_with_one_rewrite_write_back_the_rewrite_only( + self, raw_rows_guardrail, mock_request_data_input + ): + tool_call_row = { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}, "index": 0} + ], + } + original_rows = [ + {"role": "user", "content": "Look up 123-45-6789 for me.", "name": "pat"}, + tool_call_row, + {"role": "tool", "tool_call_id": "call_1", "content": '{"ssn": "123-45-6789"}'}, + ] + + def echo_with_tool_output_redacted(url, json, headers, **_kwargs): + shown_rows = json["structured_messages"] + answer = MagicMock() + answer.json.return_value = { + "action": "GUARDRAIL_INTERVENED", + "texts": ["Look up 123-45-6789 for me."], + "structured_messages": [ + shown_rows[0], + shown_rows[1], + {**shown_rows[2], "content": '{"ssn": "[REDACTED]"}'}, + ], + } + answer.raise_for_status = MagicMock() + return answer + + with patch.object(raw_rows_guardrail.async_handler, "post", side_effect=echo_with_tool_output_redacted): + guardrailed_inputs = await raw_rows_guardrail.apply_guardrail( + inputs={"texts": ["Look up 123-45-6789 for me."], "structured_messages": original_rows}, + request_data=mock_request_data_input, + input_type="request", + ) + + returned_rows = guardrailed_inputs["structured_messages"] + assert returned_rows[0] == original_rows[0] + assert returned_rows[1] == tool_call_row + assert returned_rows[2] == {"role": "tool", "tool_call_id": "call_1", "content": '{"ssn": "[REDACTED]"}'} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "row", + [ + {"role": "user", "content": "Look up 123-45-6789 for me.", "name": "pat"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}, "index": 0} + ], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "123-45-6789"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "lookup", "input": {}}]}, + { + "role": "user", + "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAA="}}], + }, + { + "role": "user", + "content": [ + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "ssn"}} + ], + }, + ], + ids=["extra_key", "tool_call_index", "tool_result_block", "tool_use_block", "image_block", "document_block"], + ) + async def test_rows_reach_the_guardrail_verbatim_when_forwarded_as_sent( + self, raw_rows_guardrail: GenericGuardrailAPI, row + ): + """Rows are the provider's own shape, so the guardrail must see every key and + content block as sent, not a copy filtered through the chat completion types.""" + assert await _structured_messages_sent(raw_rows_guardrail, [row]) == [[row]] + @pytest.mark.asyncio async def test_rows_all_echoed_back_as_shown_leave_the_rewrite_to_texts( self, generic_guardrail, mock_request_data_input @@ -672,6 +858,126 @@ class TestStructuredMessagesInResponse: assert "structured_messages" not in guardrailed_inputs assert guardrailed_inputs["texts"] == ["Never repeat an SSN.", "Look up [REDACTED] for me."] + @pytest.mark.asyncio + async def test_rows_echoed_without_the_keys_the_server_does_not_know_leave_the_rewrite_to_texts( + self, raw_rows_guardrail, mock_request_data_input + ): + """A server that parses rows with its own message types echoes them back without + the keys those types lack, and its texts redaction must still reach the LLM.""" + original_rows = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}, "index": 0} + ], + "refusal": None, + "annotations": [], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "found"}, + {"role": "user", "content": "my ssn is 123-45-6789"}, + ] + + def echo_known_keys_and_redact_texts(url, json, headers, **_kwargs): + rows = json["structured_messages"] + answer = MagicMock() + answer.json.return_value = { + "action": "GUARDRAIL_INTERVENED", + "texts": [text.replace("123-45-6789", "[SSN]") for text in json["texts"]], + "structured_messages": [ + { + "role": "assistant", + "content": None, + "refusal": None, + "tool_calls": [ + {key: value for key, value in rows[0]["tool_calls"][0].items() if key != "index"} + ], + }, + rows[1], + rows[2], + ], + } + answer.raise_for_status = MagicMock() + return answer + + with patch.object(raw_rows_guardrail.async_handler, "post", side_effect=echo_known_keys_and_redact_texts): + guardrailed_inputs = await raw_rows_guardrail.apply_guardrail( + inputs={"texts": ["found", "my ssn is 123-45-6789"], "structured_messages": original_rows}, + request_data=mock_request_data_input, + input_type="request", + ) + + assert "structured_messages" not in guardrailed_inputs + assert guardrailed_inputs["texts"] == ["found", "my ssn is [SSN]"] + + @pytest.mark.asyncio + async def test_a_row_rewrite_echoes_the_other_rows_back_as_their_original_rows( + self, raw_rows_guardrail, mock_request_data_input + ): + assistant_row = { + "role": "assistant", + "content": "Looking it up.", + "annotations": [{"type": "url_citation", "url_citation": {"url": "https://example.com"}}], + } + original_rows = [assistant_row, {"role": "user", "content": "my ssn is 123-45-6789"}] + redacted_row = {"role": "user", "content": "my ssn is [SSN]"} + + def redact_one_row(url, json, headers, **_kwargs): + answer = MagicMock() + answer.json.return_value = { + "action": "GUARDRAIL_INTERVENED", + "structured_messages": [{"role": "assistant", "content": "Looking it up."}, redacted_row], + } + answer.raise_for_status = MagicMock() + return answer + + with patch.object(raw_rows_guardrail.async_handler, "post", side_effect=redact_one_row): + guardrailed_inputs = await raw_rows_guardrail.apply_guardrail( + inputs={"texts": ["my ssn is 123-45-6789"], "structured_messages": original_rows}, + request_data=mock_request_data_input, + input_type="request", + ) + + assert guardrailed_inputs["structured_messages"] == [assistant_row, redacted_row] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "returned_row", + [ + {"role": "assistant", "content": None}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + "annotations": [], + "refusal": "blocked", + }, + ], + ids=["drops_a_declared_key", "changes_a_value"], + ) + async def test_a_row_that_loses_a_declared_key_or_changes_a_value_is_a_rewrite( + self, raw_rows_guardrail, mock_request_data_input, returned_row + ): + original_row = { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + "annotations": [], + "refusal": None, + } + answer = MagicMock() + answer.json.return_value = {"action": "GUARDRAIL_INTERVENED", "structured_messages": [returned_row]} + answer.raise_for_status = MagicMock() + + with patch.object(raw_rows_guardrail.async_handler, "post", return_value=answer): + guardrailed_inputs = await raw_rows_guardrail.apply_guardrail( + inputs={"texts": [], "structured_messages": [original_row]}, + request_data=mock_request_data_input, + input_type="request", + ) + + assert guardrailed_inputs["structured_messages"] == [returned_row] + @pytest.mark.asyncio @pytest.mark.parametrize( "structured_messages", @@ -1253,6 +1559,40 @@ class TestGenericGuardrailAPIStreamingConfig: ) assert guardrail.streaming_transform_mode == "incremental_diff" + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("configured", "forwards_as_sent"), + [ + pytest.param({}, False, id="unset"), + pytest.param({"forward_messages_as_sent": ""}, False, id="blank"), + pytest.param({"forward_messages_as_sent": False}, False, id="false"), + pytest.param({"forward_messages_as_sent": "true"}, True, id="string-true"), + pytest.param({"forward_messages_as_sent": True}, True, id="true"), + ], + ) + async def test_forward_messages_as_sent_config_decides_which_rows_reach_the_guardrail( + self, configured, forwards_as_sent + ): + from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( + initialize_guardrail, + ) + from litellm.types.guardrails import LitellmParams + + litellm_params = LitellmParams.model_validate( + { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "api_base": "https://api.test.guardrail.com", + **configured, + } + ) + with patch("litellm.logging_callback_manager.add_litellm_callback"): + guardrail = initialize_guardrail(litellm_params, {"guardrail_name": "test-forward-as-sent"}) + + row = {"role": "user", "content": "hi", "name": "pat"} + expected_row = row if forwards_as_sent else {"role": "user", "content": "hi"} + assert await _structured_messages_sent(guardrail, [row]) == [[expected_row]] + def test_initialize_guardrail_forwards_streaming_flags(self): from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( initialize_guardrail, @@ -1945,6 +2285,34 @@ class TestToolSupport: assert forwarded_tools == tools + @pytest.mark.asyncio + async def test_provider_native_tools_without_a_type_reach_the_guardrail_and_come_back_unchanged( + self, generic_guardrail + ): + """Gemini tools are declared as function_declarations, googleSearch and the + like with no type key, so they must be scanned, and a server that echoes + them back must not write a type key into the request to the provider.""" + tools = [ + {"function_declarations": [{"name": "get_weather", "parameters": {"type": "object", "properties": {}}}]}, + {"googleSearch": {}}, + ] + + def echo_tools(url, json, headers, **_kwargs): + answer = MagicMock() + answer.json.return_value = {"action": "NONE", "texts": ["hi"], "tools": json["tools"]} + answer.raise_for_status = MagicMock() + return answer + + with patch.object(generic_guardrail.async_handler, "post", side_effect=echo_tools) as mock_post: + guardrailed_inputs = await generic_guardrail.apply_guardrail( + inputs={"texts": ["hi"], "tools": tools}, + request_data={}, + input_type="request", + ) + + assert mock_post.call_args.kwargs["json"]["tools"] == tools + assert guardrailed_inputs["tools"] == tools + class TestFailOnError: """Test fail_on_error: complete fail-open on any guardrail error""" @@ -2032,6 +2400,59 @@ class TestFailOnError: assert result == {"texts": ["model output"]} + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("error", "error_type", "pre_pr_text"), + [ + ( + Timeout(message="guardrail took too long", model="gpt-5", llm_provider="openai"), + "Timeout", + "Proceeding without guardrail. guardrail_name=", + ), + ( + httpx.ConnectError("connection refused", request=MagicMock()), + "ConnectError", + "Proceeding without guardrail. guardrail_name=", + ), + ( + MaskedHTTPStatusError( + httpx.HTTPStatusError( + "bad gateway", + request=httpx.Request("POST", "http://guardrail.invalid/scan"), + response=httpx.Response(502, request=httpx.Request("POST", "http://guardrail.invalid/scan")), + ) + ), + "HTTPStatusError", + "Proceeding without guardrail. http_status_code=502 guardrail_name=", + ), + ], + ids=["timeout", "connect_error", "http_status"], + ) + async def test_fail_open_log_line_names_the_error_type( + self, fail_open_guardrail, caplog: pytest.LogCaptureFixture, error, error_type, pre_pr_text + ): + """An operator grepping the fail-open line must be able to tell a timeout + from a refused connection from an upstream status without the traceback.""" + from litellm._logging import verbose_proxy_logger + + verbose_proxy_logger.addHandler(caplog.handler) + try: + with patch.object(fail_open_guardrail.async_handler, "post", side_effect=error): + await fail_open_guardrail.apply_guardrail( + inputs={"texts": ["hi"]}, + request_data={}, + input_type="request", + ) + finally: + verbose_proxy_logger.removeHandler(caplog.handler) + + fail_open_lines: Final = [ + record.getMessage() for record in caplog.records if record.levelno == logging.CRITICAL + ] + assert fail_open_lines + assert all(f"error_type={error_type}" in line.split() for line in fail_open_lines) + assert all(pre_pr_text in line for line in fail_open_lines) + @pytest.mark.asyncio async def test_response_path_valid_block_still_blocks(self, fail_open_guardrail): """On the response path too, a valid BLOCKED decision raises despite fail_on_error=False.""" diff --git a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py index 908a49946bb..2e968ec04f5 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py @@ -953,6 +953,17 @@ async def test_provider_specific_params_includes_embedding_toggle(): assert field["default_value"] is False +@pytest.mark.asyncio +async def test_provider_specific_params_offers_forward_messages_as_sent_off_by_default(): + from litellm.proxy.guardrails.guardrail_endpoints import get_provider_specific_params + + provider_params = await get_provider_specific_params() + + field = provider_params["generic_guardrail_api"]["optional_params"]["fields"]["forward_messages_as_sent"] + assert field["type"] == "boolean" + assert field["default_value"] is False + + @pytest.mark.asyncio async def test_provider_specific_params_exposes_bedrock_streaming_flags(): from litellm.proxy.guardrails.guardrail_endpoints import get_provider_specific_params diff --git a/tests/unit/proxy/hooks/test_prompt_cache_observer.py b/tests/unit/proxy/hooks/test_prompt_cache_observer.py index 3354aa457b7..ea89bb8d4a2 100644 --- a/tests/unit/proxy/hooks/test_prompt_cache_observer.py +++ b/tests/unit/proxy/hooks/test_prompt_cache_observer.py @@ -201,6 +201,10 @@ class RecordingObserver(PromptCacheObserver): await super().async_log_success_event(kwargs, response_obj, start_time, end_time) self.finished.set() + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + await super().async_log_failure_event(kwargs, response_obj, start_time, end_time) + self.finished.set() + def native_response(): return { diff --git a/tests/unit/proxy/management_endpoints/test_ui_sso.py b/tests/unit/proxy/management_endpoints/test_ui_sso.py index 765e0506990..873182a5145 100644 --- a/tests/unit/proxy/management_endpoints/test_ui_sso.py +++ b/tests/unit/proxy/management_endpoints/test_ui_sso.py @@ -2,7 +2,7 @@ import asyncio import json import logging import os -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from contextlib import ExitStack, asynccontextmanager from types import SimpleNamespace from typing import Final @@ -17,7 +17,7 @@ from fastapi import HTTPException, Request import litellm from litellm._uuid import uuid from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.proxy._types import LiteLLM_UserTable, NewUserResponse +from litellm.proxy._types import LiteLLM_UserTable, NewUserResponse, ProxyException from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.management_endpoints.sso import CustomMicrosoftSSO from litellm.proxy.management_endpoints.types import CustomOpenID @@ -26,9 +26,11 @@ from litellm.proxy.management_endpoints.ui_sso import ( MicrosoftSSOHandler, SSOAuthenticationHandler, _complete_cli_sso_callback_session, + _insert_sso_user_or_find_concurrent_insert, _setup_team_mappings, _sync_user_role_from_jwt_role_map, cli_sso_callback, + insert_sso_user, normalize_email, process_sso_jwt_access_token, sso_lookup_user_ids, @@ -1425,6 +1427,82 @@ async def test_upsert_sso_user_no_role_in_sso_response(): assert "user_role" not in call_args.kwargs["data"] +class _UserTable: + def __init__(self, *rows: LiteLLM_UserTable) -> None: + self.rows: dict[str, LiteLLM_UserTable] = {row.user_id: row for row in rows} + + async def find_first(self, where: dict[str, dict[str, str]]) -> LiteLLM_UserTable | None: + ((field, condition),) = where.items() + + def matches(stored: object) -> bool: + if condition.get("mode") == "insensitive": + return str(stored).lower() == condition["equals"].lower() + return stored == condition["equals"] + + return next((row for row in self.rows.values() if matches(getattr(row, field))), None) + + async def find_unique(self, where: dict[str, str]) -> LiteLLM_UserTable | None: + return self.rows.get(where["user_id"]) + + +def _users_db(table: _UserTable) -> SimpleNamespace: + database: Final = SimpleNamespace(litellm_usertable=table) + return SimpleNamespace(db=database, writer_db=database) + + +async def _first_sign_in(table: _UserTable, **join: object) -> NewUserResponse | LiteLLM_UserTable: + prisma: Final = _users_db(table) + with patch("litellm.proxy.proxy_server.prisma_client", prisma): + return await _insert_sso_user_or_find_concurrent_insert( + insert=insert_sso_user( + result_openid=CustomOpenID(id="sso-subject", email="person@example.com", team_ids=[]), + user_defined_values={ + "user_id": "sso-subject", + "user_email": "person@example.com", + "user_role": None, + "max_budget": None, + "budget_duration": None, + "models": [], + }, + ), + user_id="sso-subject", + prisma_client=prisma, + **join, + ) + + +@pytest.mark.asyncio +async def test_first_sign_in_signs_in_to_the_account_a_concurrent_first_sign_in_created(): + winner: Final = LiteLLM_UserTable(user_id="sso-subject", user_email="person@example.com", user_role="internal_user") + + signed_in: Final = await _first_sign_in(_UserTable(winner)) + + assert signed_in == winner + + +@pytest.mark.asyncio +async def test_first_sign_in_does_not_sign_in_to_another_account_holding_the_email(): + other: Final = LiteLLM_UserTable(user_id="someone-else", user_email="Person@example.com", user_role="proxy_admin") + + with pytest.raises(ProxyException, match=r"User with email Person@example\.com already exists"): + await _first_sign_in(_UserTable(other)) + + +@pytest.mark.asyncio +async def test_first_sign_in_joins_default_teams_the_concurrent_first_sign_in_has_not_joined_yet(monkeypatch): + monkeypatch.setattr(litellm, "default_internal_user_params", {"teams": ["team-joined", "team-pending"]}) + table: Final = _UserTable( + LiteLLM_UserTable(user_id="sso-subject", user_email="person@example.com", teams=["team-joined"]) + ) + + async def join(user: LiteLLM_UserTable, teams: Sequence[object], prisma_client: object) -> None: + table.rows[user.user_id] = user.model_copy(update={"teams": [*user.teams, *map(str, teams)]}) + + signed_in: Final = await _first_sign_in(table, join_default_teams=join) + + assert signed_in.teams == ["team-joined", "team-pending"] + + def test_get_user_email_and_id_extracts_microsoft_role(): """ Test that _get_user_email_and_id_from_result extracts user_role from Microsoft SSO. diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 88c5e2d2c1b..4770ecd5485 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -5,11 +5,10 @@ from typing import Any, Dict, Final, List from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx -import litellm import pytest from pydantic import ValidationError - +import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.pass_through.stream_assembly import extract_sse_data @@ -2644,6 +2643,58 @@ class TestRecordPartialUsageForFailure: assert "response_cost" not in logging_obj.model_call_details +class _FailureRecorder(CustomLogger): + def __init__(self): + super().__init__() + self.failure_kwargs = [] + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.failure_kwargs.append(kwargs) + + +class TestLogErrorFrameAsFailure: + """A stream closed at the provider's `event: error` frame is one failed attempt: its failure log bills the + usage streamed before the frame on the attempt-scoped copy its caller hands it, and the request's shared + logging state stays as it was, so the attempt the router opens next logs with nothing left behind.""" + + @pytest.mark.asyncio + async def test_logs_a_failure_with_the_usage_streamed_before_the_frame_and_leaves_the_request_state_alone(self): + recorder = _FailureRecorder() + logging_obj = LiteLLMLoggingObj( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hello"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-error-frame-failure", + function_id="test-error-frame-failure", + dynamic_async_failure_callbacks=[recorder], + ) + chunks = [ + *TestRecordPartialUsageForFailure()._interrupted_chunks()[:1], + TestRecordPartialUsageForFailure._sse( + "error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}} + ), + ] + + await AnthropicPassthroughLoggingHandler.log_error_frame_as_failure( + attempt_logging_obj=logging_obj.attempt_scoped_copy(), + request_body={"model": "claude-sonnet-5", "stream": True}, + all_chunks=chunks, + error_event=("overloaded_error", "Overloaded", 529), + ) + + assert len(recorder.failure_kwargs) == 1 + failure_payload = recorder.failure_kwargs[0]["standard_logging_object"] + assert failure_payload["status"] == "failure" + assert failure_payload["prompt_tokens"] == 52 + assert failure_payload["response_cost"] > 0 + assert isinstance(recorder.failure_kwargs[0]["exception"], litellm.InternalServerError) + assert "combined_usage_object" not in logging_obj.model_call_details + assert "response_cost" not in logging_obj.model_call_details + assert logging_obj.should_run_logging("async_failure") + + @pytest.mark.parametrize( ("all_chunks", "interrupted"), [ diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index b278d7dea01..8ce41129aec 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -18,6 +18,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthr from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.anthropic.pass_through.messages.streaming_iterator import aclose_superseded_attempt from litellm.proxy.pass_through_endpoints.streaming_handler import ( PassThroughStreamingHandler, ) @@ -749,6 +750,294 @@ async def test_chunk_processor_logs_failure_not_success_on_mid_stream_exception( assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout) +def _anthropic_message_start_frame(message_id: str = "msg_1", input_tokens: int = 52) -> bytes: + return _anthropic_sse( + "message_start", + { + "type": "message_start", + "message": { + "id": message_id, + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": input_tokens, "output_tokens": 1}, + }, + }, + ) + + +def _anthropic_completed_stream_tail() -> tuple[bytes, ...]: + return ( + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse( + "message_delta", + {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 3}}, + ), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ) + + +async def _settle(recorded: list[dict[str, object]]) -> None: + for _ in range(300): + if recorded: + return + await asyncio.sleep(0.01) + + +def _anthropic_overloaded_error_frame() -> bytes: + return _anthropic_sse("error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}) + + +def _anthropic_stream_of(frames: tuple[bytes, ...]): + mock = MagicMock(spec=httpx.Response) + mock.status_code = 200 + + async def _aiter_bytes(): + for frame in frames: + yield frame + + mock.aiter_bytes = _aiter_bytes + return mock + + +@pytest.mark.parametrize( + "frames, billed", + [ + pytest.param( + (_anthropic_message_start_frame(), _anthropic_overloaded_error_frame()), True, id="error-after-message-start" + ), + pytest.param((_anthropic_overloaded_error_frame(),), False, id="error-frame-only"), + ], +) +@pytest.mark.asyncio +async def test_chunk_processor_logs_a_provider_error_frame_as_a_failure_when_closed_at_the_frame(frames, billed): + """The router retries a /v1/messages stream whose provider sent `event: error` before any content by closing + it at the frame. That attempt never completed: it logs once as a failure carrying whatever usage the provider + had reported, and the request's one-shot success log stays free for the retry that shares the logging object.""" + recorder = _EventRecorder() + logging_obj = LiteLLMLoggingObj( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-error-frame", + function_id="test-error-frame", + dynamic_async_success_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder], + ) + stream = PassThroughStreamingHandler.chunk_processor( + response=_anthropic_stream_of(frames), + request_body={"model": "claude-sonnet-5", "stream": True}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.ANTHROPIC, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + ) + received = [] + async for chunk in stream: + received.append(chunk) + if b"event: error" in chunk: + break + await aclose_superseded_attempt(stream) + await _settle(recorder.failure_kwargs) + + assert received == list(frames) + assert recorder.success_kwargs == [] + assert logging_obj.model_call_details.get("has_dispatched_final_stream_success") is not True + assert "combined_usage_object" not in logging_obj.model_call_details + assert "response_cost" not in logging_obj.model_call_details + assert logging_obj.should_run_logging("async_failure") + assert len(recorder.failure_kwargs) == 1 + failure_payload = recorder.failure_kwargs[0]["standard_logging_object"] + assert failure_payload["status"] == "failure" + assert isinstance(recorder.failure_kwargs[0]["exception"], litellm.InternalServerError) + if billed: + assert failure_payload["prompt_tokens"] == 52 + assert failure_payload["response_cost"] > 0 + else: + assert failure_payload["response_cost"] == 0 + + +@pytest.mark.asyncio +async def test_chunk_processor_logs_a_stream_that_ended_at_the_provider_error_frame_through_the_success_route(): + """A provider error frame that ends a stream nobody closed early is the request's final answer, and that + stream keeps logging the way it always has: once, through the success route, with the usage it carried.""" + recorder = _EventRecorder() + logging_obj = LiteLLMLoggingObj( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-final-error-frame", + function_id="test-final-error-frame", + dynamic_async_success_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder], + ) + frames = (_anthropic_message_start_frame(), _anthropic_overloaded_error_frame()) + received = [ + chunk + async for chunk in PassThroughStreamingHandler.chunk_processor( + response=_anthropic_stream_of(frames), + request_body={"model": "claude-sonnet-5", "stream": True}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.ANTHROPIC, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + ) + ] + await _settle(recorder.success_kwargs) + + assert received == list(frames) + assert recorder.failure_kwargs == [] + success_payload = recorder.success_kwargs[0]["standard_logging_object"] + assert success_payload["status"] == "success" + assert success_payload["prompt_tokens"] == 52 + assert logging_obj.model_call_details["prompt_cache_response_complete"] is False + + +@pytest.mark.asyncio +async def test_chunk_processor_bills_a_stream_the_client_closed_at_the_provider_error_frame(): + """The Anthropic SDK raises on a provider `event: error` frame and drops the connection, so a non-retried + stream is closed at the frame by the client, not by a retry taking over. Its usage must still reach the + success route, or the tokens the provider billed never reach the key's spend.""" + recorder = _EventRecorder() + logging_obj = LiteLLMLoggingObj( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-client-closed-at-error-frame", + function_id="test-client-closed-at-error-frame", + dynamic_async_success_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder], + ) + stream = PassThroughStreamingHandler.chunk_processor( + response=_anthropic_stream_of((_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())), + request_body={"model": "claude-sonnet-5", "stream": True}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.ANTHROPIC, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + ) + async for chunk in stream: + if b"event: error" in chunk: + break + await stream.aclose() + await _settle(recorder.success_kwargs) + + assert recorder.failure_kwargs == [] + success_payload = recorder.success_kwargs[0]["standard_logging_object"] + assert success_payload["status"] == "success" + assert success_payload["prompt_tokens"] == 52 + + +@pytest.mark.asyncio +async def test_superseded_attempt_failure_log_keeps_its_own_deployment_when_the_retry_rebinds_the_logging_object(): + """The router rebinds the shared logging object to the retry's deployment right after closing the superseded + stream, before the logging worker runs. The failure log of the superseded attempt must still name the + deployment that served it, or the router's cooldown lands on the deployment serving the retry.""" + recorder = _EventRecorder() + logging_obj = LiteLLMLoggingObj( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-superseded-deployment", + function_id="test-superseded-deployment", + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="claude-sonnet-5", litellm_params={"model_info": {"id": "deployment-1"}}, optional_params={} + ) + superseded = PassThroughStreamingHandler.chunk_processor( + response=_anthropic_stream_of((_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())), + request_body={"model": "claude-sonnet-5", "stream": True}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.ANTHROPIC, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + ) + async for chunk in superseded: + if b"event: error" in chunk: + break + await aclose_superseded_attempt(superseded) + logging_obj.update_environment_variables( + model="claude-sonnet-5-retry", litellm_params={"model_info": {"id": "deployment-2"}}, optional_params={} + ) + await _settle(recorder.failure_kwargs) + + failure_kwargs = recorder.failure_kwargs[0] + assert failure_kwargs["litellm_params"]["model_info"]["id"] == "deployment-1" + assert failure_kwargs["model"] == "claude-sonnet-5" + assert logging_obj.model_call_details["litellm_params"]["model_info"]["id"] == "deployment-2" + + +@pytest.mark.asyncio +async def test_retry_success_log_carries_its_own_usage_after_a_provider_error_frame(): + """Both attempts of a retried /v1/messages stream log through the request's one logging object. The + superseded attempt's failure log must not leave the usage it consumed behind, or the retry's success + log bills the failed attempt's tokens instead of its own.""" + recorder = _EventRecorder() + logging_obj = LiteLLMLoggingObj( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-retry-own-usage", + function_id="test-retry-own-usage", + dynamic_async_success_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder], + ) + + def attempt(frames: tuple[bytes, ...]): + return PassThroughStreamingHandler.chunk_processor( + response=_anthropic_stream_of(frames), + request_body={"model": "claude-sonnet-5", "stream": True}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.ANTHROPIC, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + ) + + superseded = attempt((_anthropic_message_start_frame("msg_a1", 52), _anthropic_overloaded_error_frame())) + async for chunk in superseded: + if b"event: error" in chunk: + break + await aclose_superseded_attempt(superseded) + await _settle(recorder.failure_kwargs) + retry = attempt((_anthropic_message_start_frame("msg_a2", 7), *_anthropic_completed_stream_tail())) + async for _ in retry: + pass + await _settle(recorder.success_kwargs) + + failure_payload = recorder.failure_kwargs[0]["standard_logging_object"] + assert failure_payload["prompt_tokens"] == 52 + success_payload = recorder.success_kwargs[0]["standard_logging_object"] + assert success_payload["status"] == "success" + assert (success_payload["prompt_tokens"], success_payload["completion_tokens"]) == (7, 3) + + def _google_sse(prompt_tokens: int, completion_tokens: int, text: str) -> bytes: payload = { "candidates": [{"content": {"parts": [{"text": text}], "role": "model"}, "index": 0}], diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index b047c29243a..a7b955f5b84 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -3454,10 +3454,11 @@ class TestResponsesInputTokens: body: dict[str, Any], path: str = "/v1/responses/input_tokens", counter: AsyncMock | None = None, + llm_router: litellm.Router | None = None, ) -> tuple[Response, AsyncMock]: from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth - from litellm.proxy.response_api_endpoints.endpoints import _proxy_token_counter + from litellm.proxy.response_api_endpoints.endpoints import _proxy_llm_router, _proxy_token_counter from litellm.types.utils import TokenCountResponse token_counter_mock = ( @@ -3474,6 +3475,7 @@ class TestResponsesInputTokens: ) app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test", request_route=path) app.dependency_overrides[_proxy_token_counter] = lambda: token_counter_mock + app.dependency_overrides[_proxy_llm_router] = lambda: llm_router try: client = TestClient(app) response = client.post(path, json=body, headers={"Authorization": "Bearer sk-9876"}) @@ -3481,6 +3483,7 @@ class TestResponsesInputTokens: finally: app.dependency_overrides.pop(user_api_key_auth, None) app.dependency_overrides.pop(_proxy_token_counter, None) + app.dependency_overrides.pop(_proxy_llm_router, None) def test_string_input_returns_openai_input_tokens_shape(self): response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "Hello, how are you?"}) @@ -3594,6 +3597,48 @@ class TestResponsesInputTokens: assert response.status_code == 400, response.text error = response.json()["error"] assert error["type"] == "invalid_request_error" + assert error["param"] == "tools" + assert error["message"].startswith("tools: ") + counter.assert_not_awaited() + + def test_invalid_tools_are_dropped_under_drop_params(self) -> None: + response, counter = self._post_input_tokens( + {"model": "gpt-4o", "input": "hi", "tools": [5, {"type": "web_search"}], "drop_params": True} + ) + + assert response.status_code == 200, response.text + assert counter.await_args.kwargs["request"].tools == [{"type": "web_search"}] + + def test_malformed_tools_follow_the_counted_deployment_even_when_fallbacks_are_configured(self) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-5.2", + "litellm_params": {"model": "openai/gpt-5.2", "api_key": "sk-test", "additional_drop_params": ["tools"]}, + }, + {"model_name": "backup", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-test"}}, + ], + fallbacks=[{"gpt-5.2": ["backup"]}], + ) + + response, counter = self._post_input_tokens( + {"model": "gpt-5.2", "input": "hi", "tools": 5}, llm_router=router + ) + + assert response.status_code == 200, response.text + assert counter.await_args.kwargs["request"].tools is None + + def test_malformed_tools_for_a_model_outside_the_router_get_a_clean_400(self) -> None: + router: Final = litellm.Router( + model_list=[{"model_name": "gpt-5.2", "litellm_params": {"model": "openai/gpt-5.2", "api_key": "sk-test"}}] + ) + + response, counter = self._post_input_tokens( + {"model": "openai/gpt-5.4-mini", "input": "hi", "tools": 5}, llm_router=router + ) + + assert response.status_code == 400, response.text + assert response.json()["error"]["param"] == "tools" counter.assert_not_awaited() def test_provider_error_maps_status_code(self): diff --git a/tests/unit/proxy/test_tools_allowlist_enforcement.py b/tests/unit/proxy/test_tools_allowlist_enforcement.py index e855ef47dd5..e306f7be4f8 100644 --- a/tests/unit/proxy/test_tools_allowlist_enforcement.py +++ b/tests/unit/proxy/test_tools_allowlist_enforcement.py @@ -145,6 +145,21 @@ class TestExtractRequestToolNames: data = {"tools": [{"type": "function", "function": {"name": "x"}}]} assert extract_request_tool_names("/v1/embeddings", data) == [] + @pytest.mark.parametrize( + ("route", "data"), + [ + ("/v1/chat/completions", {"tools": 5}), + ("/v1/chat/completions", {"functions": 5}), + ("/v1/responses", {"tools": 5}), + ("/v1/messages", {"tools": 5}), + ("/generate_content", {"tools": 5}), + ("/generate_content", {"tools": [{"functionDeclarations": 5}]}), + ], + ids=["chat_tools", "chat_functions", "responses", "messages", "generate_content", "function_declarations"], + ) + def test_malformed_tool_lists_yield_no_names(self, route: str, data: dict[str, object]) -> None: + assert extract_request_tool_names(route, data) == [] + class TestCheckToolsAllowlist: """Test allowlist enforcement in auth (no DB in hot path).""" diff --git a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py index 84aa1798da4..e83efa21dcc 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py @@ -4,6 +4,7 @@ and ``_handle_logging_proxy_only_error``.""" from __future__ import annotations import asyncio +from collections.abc import Callable from datetime import datetime from types import MappingProxyType from typing import Final @@ -921,6 +922,48 @@ async def test_post_call_failure_hook_falls_back_to_body_shape_without_a_route(p assert request_data["call_type"] == "acompletion" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_type, body", + [ + ("anthropic_messages", {"model": "m", "messages": [{"role": "user", "content": "hi"}]}), + ("aresponses", {"model": "m", "input": [{"role": "user", "content": "hi"}]}), + ("acompact_responses", {"model": "m", "input": [{"role": "user", "content": "hi"}]}), + ], +) +async def test_post_call_failure_hook_keeps_the_call_type_request_setup_logged( + proxy_logging: ProxyLogging, + make_user_api_key_auth: Callable[..., UserAPIKeyAuth], + call_type: str, + body: dict[str, object], +) -> None: + """A request rejected after request setup already carries its endpoint's call type, so the body + shape must not relabel a /v1/messages rejection as a chat completion or a list-input /v1/responses + rejection as an embedding.""" + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + + proxy_logging.alert_types = [] + request_data: Final = { + **body, + "litellm_logging_obj": LiteLLMLoggingObj( + model="m", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type=call_type, + start_time=datetime.now(), + litellm_call_id=f"test_keeps_{call_type}", + function_id=f"test_keeps_{call_type}", + ), + "metadata": {}, + } + await proxy_logging.post_call_failure_hook( + request_data=request_data, + original_exception=HTTPException(status_code=400, detail="tools must be a list"), + user_api_key_dict=make_user_api_key_auth(request_route="/v1/messages"), + ) + assert request_data["call_type"] == call_type + + @pytest.mark.asyncio @pytest.mark.parametrize("route", ["/v1/files", "/files/file-abc", "/v1/containers"]) async def test_post_call_failure_hook_keeps_the_route_for_multi_operation_routes( diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 6c52d43dfba..46efe15ee03 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -287,6 +287,19 @@ class TestFilterWebSearchDeployments: assert "deployment-2" in result_ids assert "deployment-3" not in result_ids + def test_web_search_in_a_tuple_filters_unsupported(self, sample_deployments: list[dict[str, object]]) -> None: + result: Final = filter_web_search_deployments(sample_deployments, {"tools": ({"type": "web_search"},)}) + assert [d["model_info"]["id"] for d in result] == ["deployment-1", "deployment-2"] + + @pytest.mark.parametrize("tool_type", [{}, [], None, 5], ids=["object", "list", "null", "int"]) + def test_a_non_string_tool_type_is_no_web_search_request( + self, sample_deployments: list[dict[str, object]], tool_type: object + ) -> None: + result: Final = filter_web_search_deployments( + sample_deployments, {"tools": [{"type": tool_type}, {"type": "function"}]} + ) + assert result == sample_deployments + def test_web_search_preview_filters_unsupported(self, sample_deployments): """web_search_preview type should also trigger filtering""" request_kwargs = {"tools": [{"type": "web_search_preview"}]} diff --git a/tests/unit/test_assert_ci_coverage.py b/tests/unit/test_assert_ci_coverage.py index c0e5339c925..87c56f1a42b 100644 --- a/tests/unit/test_assert_ci_coverage.py +++ b/tests/unit/test_assert_ci_coverage.py @@ -9,6 +9,8 @@ the question neither covers: whether the job that globs a file then deselects it """ import importlib.util +import shutil +import subprocess import sys from pathlib import Path from typing import Final @@ -71,17 +73,27 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path: workflow.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": "pytest tests/integration"}]}}})) _, findings = coverage._integration_ownership(tmp_path) assert [(finding.subject, finding.detail) for finding in findings] == [ - (test_path, "integration contract is also selected by GitHub Actions") + (test_path, "integration file outside GITHUB_FILES is selected by GitHub Actions") ] - github_path: Final = "tests/integration/management/test_github_contract.py" + github_path: Final = "tests/integration/database/image/test_github_contract.py" + (tmp_path / github_path).parent.mkdir(parents=True) (tmp_path / github_path).write_text("def test_contract(): pass\n") runner: Final = tmp_path / "tests/integration/run.py" runner.write_text(runner.read_text() + f"GITHUB_FILES: Final = frozenset({{{github_path!r}}})\n") - workflow.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}})) + workflow.unlink() + dedicated: Final = tmp_path / ".github/workflows/image-scan.yml" + dedicated.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}})) github_owned, github_findings = coverage._integration_ownership(tmp_path) assert github_owned == frozenset({test_path, github_path}) assert github_findings == () - workflow.write_text(yaml.safe_dump({"jobs": {}})) + per_pr: Final = tmp_path / ".github/workflows/test-mcp-interactions.yml" + per_pr.write_text(yaml.safe_dump({"jobs": {"mcp": {"steps": [{"run": f"pytest {github_path}"}]}}})) + _, shared_findings = coverage._integration_ownership(tmp_path) + assert [(finding.subject, finding.detail) for finding in shared_findings] == [ + (github_path, "GitHub-owned integration contract runs outside a dedicated integration workflow") + ] + per_pr.unlink() + dedicated.write_text(yaml.safe_dump({"jobs": {}})) _, missing_invocation = coverage._integration_ownership(tmp_path) assert [(finding.subject, finding.detail) for finding in missing_invocation] == [ (github_path, "GitHub-owned integration contract has no invoking job") @@ -94,6 +106,72 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path: assert circle_invocation == () +def _write_integration_tree(root: Path, github_path: str) -> None: + contracts: Final = root / "tests/integration/management" + contracts.mkdir(parents=True) + (contracts / "test_contract.py").write_text("def test_contract(): pass\n") + (root / github_path).parent.mkdir(parents=True, exist_ok=True) + (root / github_path).write_text("def test_contract(): pass\n") + (root / "tests/integration/run.py").write_text( + "from types import MappingProxyType\nfrom typing import Final\n" + 'GROUPS: Final = MappingProxyType({"management": ("management",)})\n' + f"GITHUB_FILES: Final = frozenset({{{github_path!r}}})\n" + ) + (root / ".circleci").mkdir() + (root / ".circleci/config.yml").write_text( + yaml.safe_dump( + { + "jobs": { + "integration_contracts": { + "steps": [{"run": {"command": "bash .circleci/scripts/run_integration.sh management"}}] + } + }, + "workflows": {"integration": {"jobs": [{"integration_contracts": {"suite": "management"}}]}}, + } + ) + ) + (root / ".github/workflows").mkdir(parents=True) + + +@pytest.mark.parametrize( + ("github_path", "workflow_name", "exit_code"), + [ + ("tests/integration/database/image/test_github_contract.py", "image-scan.yml", 0), + ("tests/integration/database/image/test_github_contract.py", "test-unit.yml", 1), + ("tests/integration/management/test_github_contract.py", "image-scan.yml", 1), + ], +) +def test_census_fails_when_a_github_owned_contract_runs_outside_its_dedicated_workflow( + tmp_path: Path, github_path: str, workflow_name: str, exit_code: int +) -> None: + script: Final = tmp_path / ".github/scripts/assert_ci_coverage.py" + script.parent.mkdir(parents=True) + shutil.copy(_MODULE_PATH, script) + _write_integration_tree(tmp_path, github_path) + (tmp_path / ".github/workflows" / workflow_name).write_text( + yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}}) + ) + result: Final = subprocess.run([sys.executable, "-I", str(script)], capture_output=True, text=True, check=False) + assert result.returncode == exit_code, result.stdout + assert "Playwright" not in result.stdout + + +def test_an_integration_file_outside_every_group_cannot_run_on_a_github_shard(tmp_path: Path) -> None: + _write_integration_tree(tmp_path, "tests/integration/database/image/test_github_contract.py") + ungrouped: Final = "tests/integration/ci_probe/test_ci_probe.py" + (tmp_path / ungrouped).parent.mkdir(parents=True) + (tmp_path / ungrouped).write_text("def test_probe(): pass\n") + _, before = coverage._integration_ownership(tmp_path) + (tmp_path / ".github/workflows/test-unit.yml").write_text( + yaml.safe_dump({"jobs": {"mcp": {"steps": [{"run": f"pytest {ungrouped}"}]}}}) + ) + _, after = coverage._integration_ownership(tmp_path) + assert [(finding.subject, finding.detail) for finding in after if finding.subject == ungrouped] == [ + (ungrouped, "integration file outside GITHUB_FILES is selected by GitHub Actions") + ] + assert not [finding for finding in before if finding.subject == ungrouped] + + def test_circleci_postgres_suites_count_their_test_path_as_invoked() -> None: config: Final = { "workflows": { diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 6bc52ab3dfc..50a9d98385d 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -38,6 +38,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.plumbing.context import set_request_root_span from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.llms.anthropic.pass_through.messages.streaming_iterator import closing_superseded_attempt from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, ) @@ -9739,6 +9740,96 @@ def test_update_kwargs_with_deployment_merge_tools_request_overrides_tool_choice assert kwargs["tool_choice"] == "none" +@pytest.mark.parametrize( + "deployment_params, router_defaults, request_params, expected_tools", + [ + ({"drop_params": True}, {}, {}, [{"type": "web_search"}]), + ({}, {"drop_params": True}, {}, [{"type": "web_search"}]), + ({}, {}, {"additional_drop_params": ["tools"]}, [{"type": "web_search"}]), + ({"drop_params": True}, {}, {"drop_params": False}, 5), + ({}, {}, {}, 5), + ], + ids=[ + "deployment_drop_params", + "router_default_drop_params", + "request_drops_tools", + "request_overrides_deployment_drop", + "no_drop_policy", + ], +) +def test_update_kwargs_with_deployment_keeps_deployment_tools_when_malformed_request_tools_drop( + deployment_params: dict[str, object], + router_defaults: dict[str, object], + request_params: dict[str, object], + expected_tools: object, +) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "o3-deep-research", + "litellm_params": { + "model": "openai/o3-deep-research", + "api_key": "fake-key", + "tools": [{"type": "web_search"}], + **deployment_params, + }, + }, + ], + default_litellm_params={**router_defaults}, + ) + kwargs: Final[dict[str, object]] = {"metadata": {}, "tools": 5, **request_params} + router._update_kwargs_with_deployment( + deployment=router.get_deployment_by_model_group_name(model_group_name="o3-deep-research"), kwargs=kwargs + ) + + assert kwargs["tools"] == expected_tools + + +_NAMED_REQUEST_TOOL_CHOICE: Final = {"type": "function", "function": {"name": "lookup_order"}} + + +@pytest.mark.parametrize( + "deployment_params, expected_tool_fields", + [ + ( + {"drop_params": True, "tool_choice": "required"}, + {"tools": [{"type": "web_search"}], "tool_choice": "required"}, + ), + ({"drop_params": True}, {"tools": [{"type": "web_search"}]}), + ({}, {"tools": 5, "tool_choice": _NAMED_REQUEST_TOOL_CHOICE, "parallel_tool_calls": False}), + ], + ids=["deployment_tool_choice_applies", "no_deployment_tool_choice", "no_drop_policy"], +) +def test_update_kwargs_with_deployment_drops_request_tool_settings_with_malformed_request_tools( + deployment_params: dict[str, object], expected_tool_fields: dict[str, object] +) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "o3-deep-research", + "litellm_params": { + "model": "openai/o3-deep-research", + "api_key": "fake-key", + "tools": [{"type": "web_search"}], + **deployment_params, + }, + }, + ], + ) + kwargs: Final[dict[str, object]] = { + "metadata": {}, + "tools": 5, + "tool_choice": _NAMED_REQUEST_TOOL_CHOICE, + "parallel_tool_calls": False, + } + router._update_kwargs_with_deployment( + deployment=router.get_deployment_by_model_group_name(model_group_name="o3-deep-research"), kwargs=kwargs + ) + + tool_fields: Final = ("tools", "tool_choice", "parallel_tool_calls") + assert {name: kwargs[name] for name in tool_fields if name in kwargs} == expected_tool_fields + + def test_credential_name_injected_as_tag(): """ Test that litellm_credential_name from deployment litellm_params @@ -16004,6 +16095,7 @@ class _AnthropicMessagesFakeByteStream: self._chunks = list(chunks) self._hidden_params = {"additional_headers": {"x-amzn-requestid": "req-1"}} self.closed = False + self.closed_as_superseded = False def __aiter__(self): return self @@ -16014,6 +16106,7 @@ class _AnthropicMessagesFakeByteStream: return self._chunks.pop(0) async def aclose(self) -> None: + self.closed_as_superseded = self.closed_as_superseded or closing_superseded_attempt() self.closed = True @@ -17256,6 +17349,30 @@ async def test_anthropic_messages_stream_dropped_before_content_is_retried_withi assert [(attempted, budget) for _, attempted, budget in provider.calls] == [(0, 2), (1, 2)] +@pytest.mark.asyncio +async def test_anthropic_messages_superseded_stream_is_closed_before_the_retry_opens(): + """Both attempts log through the request's one logging object, so the superseded attempt's end-of-stream + logging must be queued before the retry starts producing its own: the router closes the dropped stream + before it asks the group for another.""" + router = _anthropic_messages_retry_router(num_retries=1) + superseded = _AnthropicMessagesFakeByteStream( + [_anthropic_messages_message_start_chunk(), _anthropic_messages_overloaded_error_chunk()] + ) + superseded_closed_when_retry_opened: list[bool] = [] + + def retry_stream(): + superseded_closed_when_retry_opened.append(superseded.closed_as_superseded) + return _anthropic_messages_retried_stream() + + provider = _AnthropicMessagesScriptedProvider(lambda: superseded, retry_stream) + + stream = await _anthropic_messages_stream_through_router(router, provider) + body = [chunk async for chunk in stream] + + assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("pong")] + assert superseded_closed_when_retry_opened == [True] + + @pytest.mark.asyncio async def test_a_ptu_share_holders_messages_stream_dropped_before_content_is_retried_on_its_shared_group(monkeypatch): """The same-group retry has to look the group's health up as the calling team; looked up with no @@ -18387,7 +18504,7 @@ async def test_aanthropic_messages_recover_stream_error_flushes_buffered_frames_ with patch.object(router, "_aanthropic_messages_fallback_attempt") as mock_attempt: recovery = router._aanthropic_messages_recover_stream_error( - declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper() + declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() ) with pytest.raises(BedrockError) as exc_info: await drain(recovery) @@ -18408,7 +18525,7 @@ async def test_aanthropic_messages_recover_stream_error_hands_converted_raise_to with patch.object(router, "_aanthropic_messages_fallback_attempt", new=fake_attempt): recovery = router._aanthropic_messages_recover_stream_error( - raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper() + raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock() ) collected = [chunk async for chunk in recovery] assert collected == [b"fallback"] @@ -18466,6 +18583,25 @@ async def test_anthropic_messages_post_first_chunk_error_skips_fallback(): mock_fallback.assert_not_awaited() +@pytest.mark.asyncio +async def test_anthropic_messages_stream_the_client_closes_at_a_forwarded_error_is_not_closed_as_superseded(): + """An error frame forwarded to the client ends that request, and the SDK hangs up on it. No retry took the + request over, so the source must not be closed as a superseded attempt, which would log it as a failure and + drop the usage the provider billed for it.""" + router = _anthropic_messages_make_router() + error_chunk = _anthropic_messages_overloaded_error_chunk() + source = _AnthropicMessagesFakeByteStream([_anthropic_messages_content_chunk("partial answer"), error_chunk]) + + wrapped = await router._aanthropic_messages_streaming_iterator(response=source, initial_kwargs={"model": "primary"}) + async for chunk in wrapped: + if chunk == error_chunk: + break + await wrapped.aclose() + + assert source.closed is True + assert source.closed_as_superseded is False + + @pytest.mark.asyncio async def test_anthropic_messages_non_retriable_error_flushes_buffered_lifecycle_frames(): """A non-retriable error arriving while lifecycle frames are still diff --git a/tests/unit/vector_stores/test_vector_store_registry.py b/tests/unit/vector_stores/test_vector_store_registry.py index f6c888d20ff..83b59d77aaf 100644 --- a/tests/unit/vector_stores/test_vector_store_registry.py +++ b/tests/unit/vector_stores/test_vector_store_registry.py @@ -349,6 +349,56 @@ async def test_malformed_vector_store_index_row_from_the_db_raises_a_validation_ await VectorStoreIndexRegistry.get_vector_store_indexes_from_db(_database_listing_index_rows([row])) +@pytest.mark.parametrize( + ("tools", "expected"), + [ + (5, []), + (["vector_store_ids", {"type": "file_search", "vector_store_ids": ["vs_kept"]}], ["vs_kept"]), + ( + [{"type": "file_search", "vector_store_ids": 5}, {"type": "file_search", "vector_store_ids": ["vs_kept"]}], + ["vs_kept"], + ), + ], + ids=["int", "string_entry", "int_vector_store_ids"], +) +def test_vector_store_ids_to_run_skips_malformed_tools(tools: object, expected: list[str]) -> None: + assert VectorStoreRegistry().get_vector_store_ids_to_run(non_default_params={}, tools=tools) == expected + + +@pytest.mark.parametrize( + "vector_store_ids", + ["vs_requested", ["vs_requested", 5]], + ids=["string", "list_with_a_non_string"], +) +def test_vector_store_ids_to_run_reads_every_store_id_a_malformed_vector_store_ids_field_names( + vector_store_ids: object, +) -> None: + tools: Final = [{"type": "file_search", "vector_store_ids": vector_store_ids}] + assert VectorStoreRegistry().get_vector_store_ids_to_run(non_default_params={}, tools=tools) == ["vs_requested"] + + +@pytest.mark.parametrize( + "vector_store_ids", + ["vs_allowed", ["vs_allowed", 12345], [12345], ["vs_allowed"]], + ids=["string", "list_with_a_non_string", "only_non_strings", "list_of_strings"], +) +def test_vector_store_tools_pop_exactly_the_store_ids_the_registry_reports_to_run(vector_store_ids: object) -> None: + now: Final = datetime.now(timezone.utc) + registry: Final = VectorStoreRegistry( + [ + LiteLLM_ManagedVectorStore(vector_store_id=vector_store_id, created_at=now, updated_at=now) + for vector_store_id in ("vs_allowed", 12345) + ] + ) + tools: Final = [{"type": "file_search", "vector_store_ids": vector_store_ids}] + checked: Final = registry.get_vector_store_ids_to_run(non_default_params={}, tools=tools) + ran: Final[list[str]] = [] + + registry.get_and_pop_recognised_vector_store_tools(tools=[dict(tool) for tool in tools], vector_store_ids=ran) + + assert ran == checked + + class TestRegistriesDoNotShareDefaultState: def test_adding_to_one_registry_leaves_the_next_one_empty(self): VectorStoreRegistry().add_vector_store_to_registry( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx index 3eb80d3f587..e6cd9eb54a5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/PassthroughAuthorizeSection.tsx @@ -3,9 +3,11 @@ import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { Label } from "@/components/ui/label"; import { PasswordInput } from "@/components/shared/PasswordInput"; +import { Input } from "@/components/ui/input"; +import { MultiSelect } from "@/components/shared/MultiSelect"; import DcrBridgeToggle from "./DcrBridgeToggle"; import { MountedFormField } from "@/components/common_components/MountedFormField"; -import { textControl } from "./mcpFieldRules"; +import { tagsControl, textControl } from "./mcpFieldRules"; import { credentialAuthClass, isClientForwardedTokenMode } from "@/components/mcp_tools/types"; interface PassthroughOAuthFlow { @@ -69,6 +71,7 @@ export default function PassthroughAuthorizeSection({ const clientSecretPlaceholder = classUnchanged ? "Leave blank to keep the currently saved secret (if any)" : "Leave blank for public clients / PKCE"; + const fieldClassName = "rounded-lg border-border focus:border-info focus:ring-ring"; const clientIdExtra = classUnchanged ? "Set this to make everyone authorize through a specific app; required for upstreams without dynamic client registration (e.g. a pre-registered Slack app)." : "Switching the auth type discards the previously saved app; enter a client ID here or leave blank to use dynamic client registration."; @@ -113,6 +116,39 @@ export default function PassthroughAuthorizeSection({ /> )} + Authorization URL (optional)} + name="authorization_url" + help="Set this when the upstream publishes no OAuth metadata, for example an internal server behind Microsoft Entra. Leave blank to discover it from the upstream." + > + {(control) => ( + + )} + + Token URL (optional)} + name="token_url" + help="Token endpoint of the same authorization server. Leave blank to discover it from the upstream." + > + {(control) => ( + + )} + + Scopes (optional)} + name={["credentials", "scopes"]} + help="Scopes requested from the upstream when a user authorizes. Press enter or comma between scopes." + > + {(control) => } + {isEditing && onRemoveStoredAppChange && (