mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
merge main into litellm_web_search_deployment_filter
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
commit
b7a1987c33
76 changed files with 6584 additions and 245 deletions
58
.github/scripts/assert_ci_coverage.py
vendored
58
.github/scripts/assert_ci_coverage.py
vendored
|
|
@ -32,6 +32,7 @@ IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)")
|
|||
DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
|
||||
COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
|
||||
GLOB_CHARS = frozenset("*?")
|
||||
DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"})
|
||||
|
||||
# Trees whose jobs are sharded with no catch-all bucket, so every child that holds
|
||||
# tests has to be named by some shard or it runs nowhere. A child listed here is
|
||||
|
|
@ -560,6 +561,17 @@ def _integration_github_files(runner: pathlib.Path) -> frozenset[str]:
|
|||
return frozenset(ast.literal_eval(values))
|
||||
|
||||
|
||||
def _owns(workflow: str, relative_path: str) -> bool:
|
||||
owned_root: Final = DEDICATED_INTEGRATION_WORKFLOWS.get(workflow)
|
||||
return owned_root is not None and relative_path.startswith(owned_root)
|
||||
|
||||
|
||||
def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]:
|
||||
return frozenset[str]().union(
|
||||
*(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path))
|
||||
)
|
||||
|
||||
|
||||
def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
|
||||
runner: Final = repo_root / "tests/integration/run.py"
|
||||
if not runner.exists():
|
||||
|
|
@ -611,15 +623,20 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
|
|||
)
|
||||
!= 1
|
||||
)
|
||||
gha_tokens: Final = _invoked_test_tokens(
|
||||
scalar
|
||||
for path in (repo_root / ".github/workflows").glob("*.y*ml")
|
||||
for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
|
||||
workflow_tokens: Final = MappingProxyType(
|
||||
{
|
||||
path.name: _invoked_test_tokens(_scalars(yaml.safe_load(path.read_text()), path.name))
|
||||
for path in (repo_root / ".github/workflows").glob("*.y*ml")
|
||||
}
|
||||
)
|
||||
gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values())
|
||||
circle_owned: Final = (
|
||||
frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files
|
||||
)
|
||||
findings: Final = (
|
||||
tuple(
|
||||
Finding(path, "integration contract is also selected by GitHub Actions")
|
||||
for path in paths
|
||||
Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions")
|
||||
for path in sorted(circle_owned)
|
||||
if any(_token_covers(token, path) for token in gha_tokens)
|
||||
)
|
||||
+ tuple(
|
||||
|
|
@ -632,6 +649,11 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
|
|||
for path in sorted(github_files)
|
||||
if not (repo_root / path).is_file()
|
||||
)
|
||||
+ tuple(
|
||||
Finding(path, "GitHub-owned integration contract runs outside a dedicated integration workflow")
|
||||
for path in sorted(github_files)
|
||||
if any(_token_covers(token, path) for token in _tokens_outside_owner(path, workflow_tokens))
|
||||
)
|
||||
)
|
||||
browser_commands: Final = tuple(
|
||||
scalar.value
|
||||
|
|
@ -688,13 +710,9 @@ def main() -> int:
|
|||
scalars = _all_scalars()
|
||||
|
||||
integration_paths, ownership_findings = _integration_ownership()
|
||||
test_findings = (
|
||||
_uncovered_tests(
|
||||
allowlist,
|
||||
_invoked_selections(scalars)
|
||||
+ (Selection(included=integration_paths, ignored=frozenset()),),
|
||||
)
|
||||
+ ownership_findings
|
||||
test_findings = _uncovered_tests(
|
||||
allowlist,
|
||||
_invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),),
|
||||
)
|
||||
dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
|
||||
stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
|
||||
|
|
@ -711,13 +729,25 @@ def main() -> int:
|
|||
test_findings,
|
||||
"Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.",
|
||||
)
|
||||
if ownership_findings:
|
||||
browser_remedy = (
|
||||
" CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may "
|
||||
"select them and the default Playwright config must exclude them."
|
||||
if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings)
|
||||
else ""
|
||||
)
|
||||
_report(
|
||||
"integration contracts with the wrong CI owner",
|
||||
ownership_findings,
|
||||
"Follow tests/integration/AGENTS.md, which says which CI job owns each integration file." + browser_remedy,
|
||||
)
|
||||
if dockerfile_findings:
|
||||
_report(
|
||||
"Dockerfiles that no CI job builds",
|
||||
dockerfile_findings,
|
||||
"Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
|
||||
)
|
||||
if stale_findings or test_findings or dockerfile_findings:
|
||||
if stale_findings or test_findings or ownership_findings or dockerfile_findings:
|
||||
return 1
|
||||
|
||||
_write(
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a
|
|||
|
||||
`tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
|
||||
|
||||
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md`
|
||||
End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
|
||||
|
||||
When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
|
||||
|
||||
|
|
@ -72,7 +72,7 @@ When referencing or running models (coding, QA'ing, writing docs, writing tests,
|
|||
|
||||
Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
|
||||
|
||||
If you're an internal contributor, when creating a new PR, the typical flow is to branch off the repository's current default branch and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
|
||||
Internal contributors branch a new PR off the repository's current default branch, with a branch name prefixed with litellm_
|
||||
|
||||
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch
|
||||
|
||||
|
|
|
|||
|
|
@ -1416,6 +1416,7 @@ from .exceptions import (
|
|||
InvalidRequestError,
|
||||
BadRequestError,
|
||||
ImageFetchError,
|
||||
MalformedRequestFieldError as MalformedRequestFieldError,
|
||||
VectorStoreSearchError,
|
||||
NotFoundError,
|
||||
PermissionDeniedError,
|
||||
|
|
|
|||
|
|
@ -1859,6 +1859,7 @@ SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_L
|
|||
PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25))
|
||||
PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0))
|
||||
PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0))
|
||||
TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS: Final = 30
|
||||
DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = get_env_int("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60) # 1 minute
|
||||
PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)
|
||||
RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, get_env_int("RESET_BUDGET_JOB_BATCH_SIZE", 500))
|
||||
|
|
|
|||
|
|
@ -313,6 +313,10 @@ class VectorStoreSearchError(BadRequestError):
|
|||
)
|
||||
|
||||
|
||||
class MalformedRequestFieldError(BadRequestError):
|
||||
pass
|
||||
|
||||
|
||||
class UnprocessableEntityError(openai.UnprocessableEntityError):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from litellm.constants import (
|
|||
PROXY_REJECTED_BEFORE_ROUTING_KEY,
|
||||
)
|
||||
from litellm.exceptions import (
|
||||
MalformedRequestFieldError,
|
||||
validate_rate_limit_category,
|
||||
validate_rate_limit_type,
|
||||
)
|
||||
|
|
@ -3104,7 +3105,11 @@ class PrometheusLogger(CustomLogger):
|
|||
|
||||
# On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"]
|
||||
# into requested_model and leave deployment-scoped labels empty.
|
||||
deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
|
||||
deployment_selected: Final = (
|
||||
bool(model_id)
|
||||
and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
|
||||
and not isinstance(exception, MalformedRequestFieldError)
|
||||
)
|
||||
if deployment_selected:
|
||||
label_litellm_model_name = litellm_model_name
|
||||
label_model_id = model_id
|
||||
|
|
|
|||
|
|
@ -2300,6 +2300,13 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.model_call_details["combined_usage_object"] = usage
|
||||
self.model_call_details["response_cost"] = response_cost
|
||||
|
||||
def attempt_scoped_copy(self) -> "Logging":
|
||||
"""A copy with its own model_call_details, so logging one attempt of a request through it leaves
|
||||
the state the request's other attempts share untouched."""
|
||||
attempt: Final = copy.copy(self)
|
||||
attempt.model_call_details = dict(self.model_call_details)
|
||||
return attempt
|
||||
|
||||
def carry_failed_attempt_usage(self) -> None:
|
||||
"""Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
|
||||
own usage while the request's failure log still bills every attempt."""
|
||||
|
|
|
|||
|
|
@ -858,9 +858,9 @@ class StandardBuiltInToolCostTracking:
|
|||
|
||||
@staticmethod
|
||||
def get_web_search_options(kwargs: Mapping[str, object]) -> WebSearchOptions | None:
|
||||
if "web_search_options" in kwargs:
|
||||
web_search_options: Final = cast(WebSearchOptions, kwargs.get("web_search_options", {}))
|
||||
return WebSearchOptions(**web_search_options)
|
||||
web_search_options: Final = kwargs.get("web_search_options")
|
||||
if isinstance(web_search_options, dict):
|
||||
return WebSearchOptions(**cast(WebSearchOptions, web_search_options))
|
||||
|
||||
tools: Final = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
|
||||
kwargs=kwargs, tool_type="web_search_preview"
|
||||
|
|
@ -876,10 +876,9 @@ class StandardBuiltInToolCostTracking:
|
|||
_get_web_search_options = get_web_search_options
|
||||
|
||||
@staticmethod
|
||||
def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> list[object] | None:
|
||||
if "tools" in kwargs:
|
||||
return cast(list[object], kwargs.get("tools", []))
|
||||
return None
|
||||
def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> Sequence[object] | None:
|
||||
tools: Final = kwargs.get("tools")
|
||||
return cast(Sequence[object], tools) if isinstance(tools, (list, tuple)) else None
|
||||
|
||||
@staticmethod
|
||||
def get_file_search_tool_call(kwargs: Mapping[str, object]) -> FileSearchTool | None:
|
||||
|
|
|
|||
164
litellm/litellm_core_utils/tool_fields.py
Normal file
164
litellm/litellm_core_utils/tool_fields.py
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from typing import Annotated, Final, TypeAlias
|
||||
|
||||
from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
|
||||
ROUTES_WITH_TOOLS: Final = frozenset(
|
||||
{
|
||||
"acompletion",
|
||||
"aresponses",
|
||||
"acompact_responses",
|
||||
"anthropic_messages",
|
||||
"agenerate_content",
|
||||
"agenerate_content_stream",
|
||||
"acreate_interaction",
|
||||
}
|
||||
)
|
||||
TOOL_FIELDS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls", "toolConfig", "tool_config"})
|
||||
_TOOLS: Final = TypeAdapter(list[object])
|
||||
_FIELD_LIST: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
|
||||
_JSON_OBJECT: Final = TypeAdapter(Json[dict[str, object]])
|
||||
|
||||
|
||||
def _schema_as_read(value: object) -> object:
|
||||
if not value:
|
||||
return None
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
try:
|
||||
return _JSON_OBJECT.validate_python(value)
|
||||
except ValidationError:
|
||||
return value
|
||||
|
||||
|
||||
_Schema: TypeAlias = Annotated[dict[str, object] | None, BeforeValidator(_schema_as_read)]
|
||||
|
||||
|
||||
class _ToolFunction(TypedDict, total=False):
|
||||
parameters: ReadOnly[_Schema]
|
||||
|
||||
|
||||
class _ToolEntry(TypedDict, total=False):
|
||||
function: ReadOnly[_ToolFunction | None]
|
||||
parameters: ReadOnly[_Schema]
|
||||
input_schema: ReadOnly[_Schema]
|
||||
vector_store_ids: ReadOnly[list[str] | None]
|
||||
|
||||
|
||||
_TOOL_ENTRY: Final = TypeAdapter(_ToolEntry)
|
||||
_WEB_SEARCH_OPTIONS: Final = TypeAdapter(dict[str, object] | None, config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DropPolicy:
|
||||
drop_params: bool
|
||||
additional_drop_params: frozenset[str]
|
||||
|
||||
def drops(self, field: str) -> bool:
|
||||
return self.drop_params or field in self.additional_drop_params
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MalformedField:
|
||||
field: str
|
||||
message: str
|
||||
dropped: Callable[[Mapping[str, object]], Mapping[str, object]]
|
||||
|
||||
|
||||
def _field_names(value: object) -> frozenset[str]:
|
||||
try:
|
||||
return frozenset(name for name in _FIELD_LIST.validate_python(value) if isinstance(name, str))
|
||||
except ValidationError:
|
||||
return frozenset()
|
||||
|
||||
|
||||
def drop_policy(params: Mapping[str, object]) -> DropPolicy:
|
||||
return DropPolicy(
|
||||
drop_params=litellm.drop_params is True or normalize_drop_params(params.get("drop_params")) is True,
|
||||
additional_drop_params=_field_names(params.get("additional_drop_params")),
|
||||
)
|
||||
|
||||
|
||||
def first_validation_error(e: ValidationError, prefix: tuple[str | int, ...] = ()) -> str:
|
||||
error: Final = e.errors(include_url=False)[0]
|
||||
return f"{'.'.join(str(part) for part in (*prefix, *error['loc']))}: {error['msg']}"
|
||||
|
||||
|
||||
def _tool_entry_error(index: int, entry: object) -> str | None:
|
||||
try:
|
||||
_TOOL_ENTRY.validate_python(entry, strict=True)
|
||||
except ValidationError as e:
|
||||
return first_validation_error(e, prefix=("tools", index))
|
||||
return None
|
||||
|
||||
|
||||
def _without_tool_fields(data: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {name: value for name, value in data.items() if name not in TOOL_FIELDS}
|
||||
|
||||
|
||||
def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None:
|
||||
tools: Final = data.get("tools")
|
||||
if tools is None:
|
||||
return None
|
||||
try:
|
||||
entries: Final = _TOOLS.validate_python(tools)
|
||||
except ValidationError as e:
|
||||
return MalformedField("tools", first_validation_error(e, prefix=("tools",)), _without_tool_fields)
|
||||
entry_errors: Final = tuple(_tool_entry_error(index, entry) for index, entry in enumerate(entries))
|
||||
first_error: Final = next((error for error in entry_errors if error is not None), None)
|
||||
if first_error is None:
|
||||
return None
|
||||
kept: Final = [entry for entry, error in zip(entries, entry_errors) if error is None]
|
||||
return MalformedField(
|
||||
"tools", first_error, lambda fields: {**fields, "tools": kept} if kept else _without_tool_fields(fields)
|
||||
)
|
||||
|
||||
|
||||
def _without_web_search_options(data: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {name: value for name, value in data.items() if name != "web_search_options"}
|
||||
|
||||
|
||||
def _malformed_web_search_options(data: Mapping[str, object]) -> MalformedField | None:
|
||||
try:
|
||||
_WEB_SEARCH_OPTIONS.validate_python(data.get("web_search_options"))
|
||||
except ValidationError as e:
|
||||
return MalformedField(
|
||||
"web_search_options",
|
||||
first_validation_error(e, prefix=("web_search_options",)),
|
||||
_without_web_search_options,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def resolve_tool_fields(
|
||||
data: Mapping[str, object], candidates: Callable[[], tuple[DropPolicy, ...]]
|
||||
) -> Mapping[str, object] | MalformedField:
|
||||
checked: Final = (_malformed_tools(data), _malformed_web_search_options(data))
|
||||
malformed: Final = tuple(field for field in checked if field is not None)
|
||||
if not malformed:
|
||||
return data
|
||||
policies: Final = candidates()
|
||||
rejected: Final = next(
|
||||
(field for field in malformed if not any(policy.drops(field.field) for policy in policies)), None
|
||||
)
|
||||
if rejected is not None:
|
||||
return rejected
|
||||
return reduce(
|
||||
lambda resolved, field: field.dropped(resolved),
|
||||
(field for field in malformed if all(policy.drops(field.field) for policy in policies)),
|
||||
data,
|
||||
)
|
||||
|
||||
|
||||
def resolve_deployment_tool_fields(
|
||||
kwargs: Mapping[str, object], route: object
|
||||
) -> Mapping[str, object] | MalformedField:
|
||||
if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS:
|
||||
return kwargs
|
||||
return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),))
|
||||
|
|
@ -45,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
|
|||
effective_scan_only_tool_results_for_guardrail,
|
||||
effective_skip_system_message_for_guardrail,
|
||||
effective_skip_tool_message_for_guardrail,
|
||||
list_field,
|
||||
merge_guardrailed_scoped_messages,
|
||||
merge_returned_tools_into_request_tools,
|
||||
scoped_structured_message_indices,
|
||||
|
|
@ -924,10 +925,10 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
images=(),
|
||||
)
|
||||
|
||||
def extract_request_tool_names(self, data: dict) -> list[str]:
|
||||
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
|
||||
"""Extract every tool name in an Anthropic messages request: tools[].name, plus
|
||||
tools[].function.name for OpenAI-format tools the bridge forwards verbatim."""
|
||||
return [name for tool in data.get("tools") or [] for name in anthropic_tool_names(tool)]
|
||||
return [name for tool in list_field(data, "tools") for name in anthropic_tool_names(tool)]
|
||||
|
||||
@classmethod
|
||||
def _extract_input_text_and_images(
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import asyncio
|
|||
import json
|
||||
import re
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence
|
||||
from contextvars import ContextVar
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable
|
||||
|
||||
|
|
@ -352,6 +353,22 @@ async def aclose_if_supported(stream: object) -> None:
|
|||
await stream.aclose()
|
||||
|
||||
|
||||
_CLOSING_SUPERSEDED_ATTEMPT: Final[ContextVar[bool]] = ContextVar("closing_superseded_attempt", default=False)
|
||||
|
||||
|
||||
async def aclose_superseded_attempt(stream: object) -> None:
|
||||
"""Close an attempt a retry or fallback is taking over, so its end-of-stream logging records a failure."""
|
||||
token: Final = _CLOSING_SUPERSEDED_ATTEMPT.set(True)
|
||||
try:
|
||||
await aclose_if_supported(stream)
|
||||
finally:
|
||||
_CLOSING_SUPERSEDED_ATTEMPT.reset(token)
|
||||
|
||||
|
||||
def closing_superseded_attempt() -> bool:
|
||||
return _CLOSING_SUPERSEDED_ATTEMPT.get()
|
||||
|
||||
|
||||
_RESPONSE_HEADERS_ADAPTER: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import json
|
|||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles
|
||||
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
|
||||
from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
|
||||
|
|
@ -292,6 +292,17 @@ def openai_tool_name(tool: object) -> str | None:
|
|||
return flat_name if isinstance(flat_name, str) else None
|
||||
|
||||
|
||||
_FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True))
|
||||
_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
def list_field(data: object, key: str) -> Sequence[object]:
|
||||
try:
|
||||
return _ITEMS.validate_python(_FIELDS.validate_python(data).get(key))
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def anthropic_tool_names(tool: object) -> tuple[str, ...]:
|
||||
"""Every name a /v1/messages tool dict can act under: the flat Anthropic ``name`` plus
|
||||
``function.name`` for OpenAI-format tools the bridge forwards verbatim. Allowlist checks
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
|
|||
effective_scan_only_tool_results_for_guardrail,
|
||||
effective_skip_system_message_for_guardrail,
|
||||
effective_skip_tool_message_for_guardrail,
|
||||
list_field,
|
||||
merge_guardrailed_scoped_messages,
|
||||
merge_returned_tools_into_request_tools,
|
||||
openai_tool_name,
|
||||
|
|
@ -156,7 +157,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
inputs["structured_messages"] = [structured_messages[index] for index in scoped_message_indices]
|
||||
# Pass tools (function definitions) to the guardrail
|
||||
tools: Final = data.get("tools")
|
||||
if tools and not scan_only_tool_results:
|
||||
if isinstance(tools, list) and tools and not scan_only_tool_results:
|
||||
inputs["tools"] = tools
|
||||
# Include model information if available
|
||||
model: Final = data.get("model")
|
||||
|
|
@ -258,19 +259,19 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
return "no scannable content after message scoping"
|
||||
return None if images else "no scannable content"
|
||||
|
||||
def extract_request_tool_names(self, data: dict) -> list[str]:
|
||||
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
|
||||
"""Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name,
|
||||
and the tools a replayed tool_search_output loads)."""
|
||||
names: Final[list[str]] = []
|
||||
for tool in data.get("tools") or []:
|
||||
for tool in list_field(data, "tools"):
|
||||
if isinstance(tool, dict) and tool.get("type") == "function":
|
||||
fn = tool.get("function")
|
||||
if isinstance(fn, dict) and fn.get("name"):
|
||||
names.append(str(fn["name"]))
|
||||
for fn in data.get("functions") or []:
|
||||
for fn in list_field(data, "functions"):
|
||||
if isinstance(fn, dict) and fn.get("name"):
|
||||
names.append(str(fn["name"]))
|
||||
return [*names, *replayed_tool_search_tool_names(cast(object, data))] # cast-ok: upcast, read field by field
|
||||
return [*names, *replayed_tool_search_tool_names(data)]
|
||||
|
||||
def _extract_inputs(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
|||
from litellm.llms.base_llm.guardrail_translation.utils import (
|
||||
blocked_responses_stream_usage,
|
||||
effective_skip_system_message_for_guardrail,
|
||||
list_field,
|
||||
merge_guardrailed_scoped_messages,
|
||||
role_out_of_guardrail_scope,
|
||||
scoped_structured_message_indices,
|
||||
|
|
@ -764,11 +765,11 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
data.get("input"), data.get("instructions"), structured_messages, merged
|
||||
)
|
||||
|
||||
def extract_request_tool_names(self, data: dict) -> list[str]:
|
||||
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
|
||||
"""Extract tool names from Responses API request (tools[].name for function
|
||||
and custom, tools[].server_label for mcp)."""
|
||||
names: Final[list[str]] = []
|
||||
for tool in data.get("tools") or []:
|
||||
for tool in list_field(data, "tools"):
|
||||
if not isinstance(tool, dict):
|
||||
continue
|
||||
if tool.get("type") in ("function", "custom") and tool.get("name"):
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ from litellm.proxy._experimental.mcp_server.catalog import (
|
|||
)
|
||||
from litellm.proxy._experimental.mcp_server.contracts import OperationContext
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
from litellm.types.mcp import MCP_LEGACY_VERSIONS
|
||||
from litellm.types.mcp_server.mcp_server_manager import MCPServer
|
||||
|
||||
|
|
@ -300,6 +301,29 @@ async def _listing(event: ServerEvent, context: OperationContext) -> str | None:
|
|||
raise MCPError(code=-32603, message="Subscription catalog exceeds the pagination limit")
|
||||
|
||||
|
||||
async def _enforce_rate_limit(context: OperationContext, server: MCPServer) -> None:
|
||||
await operations._enforce_mcp_server_rate_limit(context.user_api_key_auth, server) # pyright: ignore[reportPrivateUsage] # same admission policy as other gateway operations
|
||||
|
||||
|
||||
async def _unmetered(_: OperationContext, __: MCPServer) -> None:
|
||||
return None
|
||||
|
||||
|
||||
async def _admitted() -> ProxyRateLimitError | None:
|
||||
return None
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _listener_admission(targets: tuple[SubscriptionTarget, ...]) -> AsyncGenerator[None]:
|
||||
scope: Final = operations._mcp_server_admission_memo.set( # pyright: ignore[reportPrivateUsage] # the listener's own catalog refreshes reuse its admission
|
||||
{target.server.server_id: asyncio.create_task(_admitted()) for target in targets}
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
operations._mcp_server_admission_memo.reset(scope) # pyright: ignore[reportPrivateUsage] # restore the caller's admission scope
|
||||
|
||||
|
||||
async def _recheck_authorization(check: Callable[[], Awaitable[OperationContext]]) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(15)
|
||||
|
|
@ -361,7 +385,7 @@ class GatewaySubscriptions:
|
|||
if ctx.protocol_version != "2026-07-28":
|
||||
raise MCPError(code=-32601, message="Subscriptions require the modern MCP protocol")
|
||||
async with self._capacity(owner):
|
||||
return await self._listen(ctx, params, authorize)
|
||||
return await self._listen(ctx, params, authorize, _enforce_rate_limit)
|
||||
|
||||
@asynccontextmanager
|
||||
async def _capacity(self, owner: str) -> AsyncGenerator[None]:
|
||||
|
|
@ -414,6 +438,7 @@ class GatewaySubscriptions:
|
|||
ctx,
|
||||
SubscriptionFilter(tools_list_changed=True, prompts_list_changed=True, resources_list_changed=True),
|
||||
authorization,
|
||||
_unmetered,
|
||||
)
|
||||
if (ready := self._initializing.get(key)) is not None:
|
||||
ready.set()
|
||||
|
|
@ -427,6 +452,7 @@ class GatewaySubscriptions:
|
|||
ctx: ServerRequestContext,
|
||||
notifications: SubscriptionFilter,
|
||||
authorization: SubscriptionAuthorization,
|
||||
admit: Callable[[OperationContext, MCPServer], Awaitable[None]],
|
||||
scope: anyio.CancelScope | None = None,
|
||||
*,
|
||||
task_status: TaskStatus[None],
|
||||
|
|
@ -443,7 +469,7 @@ class GatewaySubscriptions:
|
|||
|
||||
async with self._capacity(authorization.owner):
|
||||
await self._listen(
|
||||
ctx, SubscriptionsListenRequestParams(notifications=notifications), authorization, ready
|
||||
ctx, SubscriptionsListenRequestParams(notifications=notifications), authorization, admit, ready
|
||||
)
|
||||
|
||||
async def legacy_resource(self, ctx: ServerRequestContext, uri: str, *, subscribe: bool) -> None:
|
||||
|
|
@ -464,6 +490,7 @@ class GatewaySubscriptions:
|
|||
ctx,
|
||||
SubscriptionFilter(resource_subscriptions=[uri]),
|
||||
state.authorization,
|
||||
_enforce_rate_limit,
|
||||
scope,
|
||||
)
|
||||
state.resources = {**state.resources, uri: scope}
|
||||
|
|
@ -476,6 +503,7 @@ class GatewaySubscriptions:
|
|||
ctx: ServerRequestContext,
|
||||
params: SubscriptionsListenRequestParams,
|
||||
authorize: Callable[[], Awaitable[OperationContext]],
|
||||
admit: Callable[[OperationContext, MCPServer], Awaitable[None]],
|
||||
ready: Callable[[SubscriptionFilter], None] | None = None,
|
||||
) -> SubscriptionsListenResult:
|
||||
context: Final = await authorize()
|
||||
|
|
@ -501,7 +529,7 @@ class GatewaySubscriptions:
|
|||
return fresh
|
||||
|
||||
async def relay(target: SubscriptionTarget, *, task_status: TaskStatus[SubscriptionFilter]) -> None:
|
||||
await operations._enforce_mcp_server_rate_limit(context.user_api_key_auth, target.server) # pyright: ignore[reportPrivateUsage] # same admission policy as other gateway operations
|
||||
await admit(context, target.server)
|
||||
|
||||
legacy: Final = LegacyNotifications()
|
||||
|
||||
|
|
@ -567,7 +595,7 @@ class GatewaySubscriptions:
|
|||
finally:
|
||||
self._upstream_slots.release()
|
||||
|
||||
async with anyio.create_task_group() as group:
|
||||
async with _listener_admission(targets), anyio.create_task_group() as group:
|
||||
honored: Final = tuple(
|
||||
[SubscriptionFilter.model_validate(await group.start(relay, target)) for target in targets]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -14261,6 +14261,19 @@
|
|||
"description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.",
|
||||
"title": "Fail On Error"
|
||||
},
|
||||
"forward_messages_as_sent": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": false,
|
||||
"description": "Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can carry provider-native content blocks and keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with undeclared keys removed.",
|
||||
"title": "Forward Messages As Sent"
|
||||
},
|
||||
"guard_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
@ -16612,6 +16625,19 @@
|
|||
"description": "HTTP timeout in seconds for checking attached files. Default: 10.",
|
||||
"title": "File Guardrail Timeout"
|
||||
},
|
||||
"forward_messages_as_sent": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": false,
|
||||
"description": "Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can carry provider-native content blocks and keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with undeclared keys removed.",
|
||||
"title": "Forward Messages As Sent"
|
||||
},
|
||||
"gateway_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -6,9 +6,9 @@ from collections.abc import Mapping
|
|||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import JsonValue
|
||||
from pydantic import JsonValue, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -18,6 +18,7 @@ from litellm.anthropic_interface.exceptions import (
|
|||
AnthropicExceptionMapping,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.litellm_core_utils.tool_fields import first_validation_error
|
||||
from litellm.llms.anthropic.pass_through.context_management import (
|
||||
AnthropicContextManagementError,
|
||||
)
|
||||
|
|
@ -32,6 +33,7 @@ from litellm.proxy.common_request_processing import (
|
|||
log_llm_api_exception,
|
||||
proxy_exception_from_http_exception,
|
||||
proxy_exception_from_route_error,
|
||||
resolve_counted_tool_fields,
|
||||
resolve_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.error_body_call_id import error_body_call_id
|
||||
|
|
@ -47,6 +49,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
with_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
|
|
@ -91,6 +94,31 @@ def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSO
|
|||
return JSONResponse(status_code=status_code, content=content, headers=exc.headers)
|
||||
|
||||
|
||||
_TOKEN_COUNT_FIELDS: Final = ("model", "messages", "tools", "system")
|
||||
|
||||
|
||||
def _token_count_request(fields: Mapping[str, object]) -> TokenCountRequest | ProxyException:
|
||||
try:
|
||||
return TokenCountRequest.model_validate({name: fields[name] for name in _TOKEN_COUNT_FIELDS if name in fields})
|
||||
except ValidationError as e:
|
||||
return ProxyException(
|
||||
message=first_validation_error(e),
|
||||
type="invalid_request_error",
|
||||
param=str(e.errors(include_url=False)[0]["loc"][0]),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
|
||||
def _token_count_fields(
|
||||
data: Mapping[str, object], auto_drop_params: bool, llm_router: Router | None, team_id: str | None
|
||||
) -> Mapping[str, object] | ProxyException:
|
||||
fields: Final = {
|
||||
name: data[name] for name in (*_TOKEN_COUNT_FIELDS, "drop_params", "additional_drop_params") if name in data
|
||||
}
|
||||
resolving: Final = {**fields, "drop_params": True} if auto_drop_params else fields
|
||||
return resolve_counted_tool_fields(resolving, llm_router, team_id)
|
||||
|
||||
|
||||
def _strip_total_tokens_from_anthropic_response(response: object) -> None:
|
||||
"""Remove the OpenAI-flavored `usage.total_tokens` field that LiteLLM
|
||||
injects into Anthropic /v1/messages responses.
|
||||
|
|
@ -351,7 +379,7 @@ async def count_tokens(
|
|||
|
||||
Returns: {"input_tokens": <number>}
|
||||
"""
|
||||
from litellm.proxy.proxy_server import count_request_tokens, proxy_config
|
||||
from litellm.proxy.proxy_server import count_request_tokens, llm_router, proxy_config
|
||||
|
||||
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
|
||||
try:
|
||||
|
|
@ -368,26 +396,21 @@ async def count_tokens(
|
|||
if not messages:
|
||||
raise HTTPException(status_code=400, detail="messages parameter is required")
|
||||
|
||||
# Create TokenCountRequest for the internal endpoint
|
||||
from litellm.proxy._types import TokenCountRequest
|
||||
|
||||
token_request: Final = TokenCountRequest(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
tools=data.get("tools"),
|
||||
system=data.get("system"),
|
||||
auto_drop_params: Final = should_auto_drop_params_for_agentic_cli(
|
||||
request.headers.get("user-agent", ""), "drop_params" in data, proxy_config
|
||||
)
|
||||
token_fields: Final = _token_count_fields(data, auto_drop_params, llm_router, user_api_key_dict.team_id)
|
||||
token_request: Final = (
|
||||
token_fields if isinstance(token_fields, ProxyException) else _token_count_request(token_fields)
|
||||
)
|
||||
if isinstance(token_request, ProxyException):
|
||||
return _anthropic_error_json_response(token_request, request)
|
||||
|
||||
token_response: Final = await count_request_tokens(
|
||||
request=token_request,
|
||||
call_endpoint=True,
|
||||
messages_format="anthropic",
|
||||
**_count_tokens_body_options(
|
||||
data,
|
||||
should_auto_drop_params_for_agentic_cli(
|
||||
request.headers.get("user-agent", ""), "drop_params" in data, proxy_config
|
||||
),
|
||||
),
|
||||
**_count_tokens_body_options(data, auto_drop_params),
|
||||
)
|
||||
except HTTPException as e:
|
||||
return _anthropic_error_json_response(proxy_exception_from_http_exception(e, {}), request)
|
||||
|
|
|
|||
|
|
@ -80,6 +80,7 @@ from litellm.litellm_core_utils.served_output_texts import (
|
|||
from litellm.litellm_core_utils.streaming_handler import (
|
||||
backfill_missing_cache_usage_fields,
|
||||
)
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields
|
||||
from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
can_key_call_resolved_model,
|
||||
|
|
@ -129,6 +130,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
|
|||
)
|
||||
from litellm.router import Router
|
||||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.router import RouterRateLimitError
|
||||
|
|
@ -633,6 +635,63 @@ def serialize_http_exception_detail(
|
|||
return str(detail), None
|
||||
|
||||
|
||||
_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True))
|
||||
_UNSEEN_DEPLOYMENT: Final = DropPolicy(drop_params=True, additional_drop_params=frozenset())
|
||||
|
||||
|
||||
def _is_auto_router(params: Mapping[str, object]) -> bool:
|
||||
model: Final = params.get("model")
|
||||
return isinstance(model, str) and model.startswith(AUTO_ROUTER_MODEL_PREFIX)
|
||||
|
||||
|
||||
def _named_deployments(llm_router: Router, model: str, team_id: str | None) -> tuple[Mapping[str, object], ...]:
|
||||
by_id: Final = None if model in llm_router.model_names else llm_router.get_deployment(model_id=model)
|
||||
if by_id is not None:
|
||||
return (_REQUEST_FIELDS.validate_python(by_id.litellm_params.model_dump(exclude_none=True)),)
|
||||
group: Final = llm_router.get_model_from_alias(model=model) or model
|
||||
if group not in llm_router.model_names or llm_router.get_model_list_from_routing_groups(model_name=group):
|
||||
return ()
|
||||
deployments: Final = tuple(
|
||||
_REQUEST_FIELDS.validate_python(deployment["litellm_params"])
|
||||
for deployment in llm_router.get_all_deployments(model_name=group, team_id=team_id)
|
||||
)
|
||||
return () if any(_is_auto_router(params) for params in deployments) else deployments
|
||||
|
||||
|
||||
def _router_defaults(llm_router: Router | None) -> Mapping[str, object]:
|
||||
configured: Final[object] = getattr(llm_router, "default_litellm_params", None)
|
||||
if configured is None:
|
||||
return {}
|
||||
return {name: value for name, value in _REQUEST_FIELDS.validate_python(configured).items() if value is not None}
|
||||
|
||||
|
||||
def _counted_drop_policies(
|
||||
data: Mapping[str, object], llm_router: Router | None, team_id: str | None
|
||||
) -> tuple[DropPolicy, ...]:
|
||||
requested: Final = {**_router_defaults(llm_router), **data}
|
||||
model: Final = data.get("model")
|
||||
deployments: Final = (
|
||||
_named_deployments(llm_router, model, team_id) if llm_router is not None and isinstance(model, str) else ()
|
||||
)
|
||||
if not deployments:
|
||||
return (drop_policy(requested), _UNSEEN_DEPLOYMENT)
|
||||
return tuple(drop_policy({**deployment, **requested}) for deployment in deployments)
|
||||
|
||||
|
||||
def resolve_counted_tool_fields(
|
||||
data: Mapping[str, object], llm_router: Router | None, team_id: str | None
|
||||
) -> Mapping[str, object] | ProxyException:
|
||||
resolved: Final = resolve_tool_fields(data, lambda: _counted_drop_policies(data, llm_router, team_id))
|
||||
if isinstance(resolved, MalformedField):
|
||||
return ProxyException(
|
||||
message=resolved.message,
|
||||
type="invalid_request_error",
|
||||
param=resolved.field,
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
def proxy_exception_from_http_exception(exc: HTTPException, headers: dict[str, str]) -> ProxyException:
|
||||
raw_detail: Final = attribute_of(exc, "detail", str(exc))
|
||||
message, structured_fields = serialize_http_exception_detail(raw_detail)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
|
|||
additional_provider_specific_params=getattr(litellm_params, "additional_provider_specific_params", {}),
|
||||
unreachable_fallback=getattr(litellm_params, "unreachable_fallback", "fail_closed"),
|
||||
fail_on_error=getattr(litellm_params, "fail_on_error", True),
|
||||
forward_messages_as_sent=getattr(litellm_params, "forward_messages_as_sent", False),
|
||||
extra_headers=getattr(litellm_params, "extra_headers", None),
|
||||
guardrail_name=guardrail.get("guardrail_name", ""),
|
||||
event_hook=litellm_params.mode,
|
||||
|
|
|
|||
|
|
@ -7,10 +7,12 @@
|
|||
|
||||
import fnmatch
|
||||
import os
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final, Literal, Optional
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic_core import PydanticSerializationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._version import version as litellm_version
|
||||
|
|
@ -38,6 +40,10 @@ if TYPE_CHECKING:
|
|||
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
|
||||
|
||||
GUARDRAIL_NAME: Final = "generic_guardrail_api"
|
||||
_CHAT_ROW: Final[TypeAdapter[AllMessageValues]] = TypeAdapter(AllMessageValues)
|
||||
_SENT_ROW: Final = TypeAdapter(Mapping[str, object])
|
||||
_SENT_ROWS: Final = TypeAdapter(Sequence[Mapping[str, object]])
|
||||
_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
||||
# Headers whose values are forwarded as-is (case-insensitive). Glob patterns supported (e.g. x-stainless-*, x-litellm*).
|
||||
_HEADER_VALUE_ALLOWLIST: Final = frozenset(
|
||||
|
|
@ -150,23 +156,65 @@ def _extract_inbound_headers(
|
|||
return None
|
||||
|
||||
|
||||
def _is_contained_in(part: JsonValue, whole: JsonValue) -> bool:
|
||||
pending: Final[list[tuple[JsonValue, JsonValue]]] = [(part, whole)] # mutable-ok: worklist, recursion is banned
|
||||
while pending:
|
||||
match pending.pop():
|
||||
case dict() as part_dict, dict() as whole_dict:
|
||||
if not part_dict.keys() <= whole_dict.keys():
|
||||
return False
|
||||
pending.extend((value, whole_dict[key]) for key, value in part_dict.items())
|
||||
case list() as part_list, list() as whole_list:
|
||||
if len(part_list) != len(whole_list):
|
||||
return False
|
||||
pending.extend(zip(part_list, whole_list, strict=True))
|
||||
case part_leaf, whole_leaf if part_leaf != whole_leaf:
|
||||
return False
|
||||
case _:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def _is_shown_row(shown: AllMessageValues, returned: AllMessageValues) -> bool:
|
||||
return returned == shown
|
||||
|
||||
|
||||
def _is_echo(original: AllMessageValues, returned: AllMessageValues) -> bool:
|
||||
"""A server that parses rows with its own message types and echoes them back drops the
|
||||
keys those types do not declare. That is still an echo as long as nothing changed and
|
||||
every key the chat completion types declare is kept."""
|
||||
if returned == original:
|
||||
return True
|
||||
try:
|
||||
sent: Final[JsonValue] = _JSON.validate_json(_SENT_ROW.dump_json(original))
|
||||
declared: Final[JsonValue] = _JSON.validate_json(
|
||||
_CHAT_ROW.dump_json(_CHAT_ROW.validate_python(original), warnings="error")
|
||||
)
|
||||
echoed: Final[JsonValue] = _JSON.validate_python(returned)
|
||||
except (ValidationError, PydanticSerializationError):
|
||||
return False
|
||||
return _is_contained_in(echoed, sent) and _is_contained_in(declared, echoed)
|
||||
|
||||
|
||||
def _structured_rows_to_write_back(
|
||||
original_rows: Sequence[AllMessageValues] | None,
|
||||
shown_rows: Sequence[AllMessageValues] | None,
|
||||
compared_rows: Sequence[AllMessageValues] | None,
|
||||
returned_rows: Sequence[AllMessageValues],
|
||||
is_echo: Callable[[AllMessageValues, AllMessageValues], bool],
|
||||
) -> tuple[AllMessageValues, ...] | None:
|
||||
"""The request model drops row keys its message types do not declare, so a
|
||||
row the server echoes back verbatim is restored to the original row object.
|
||||
A server that echoes every row back unchanged has not rewritten anything
|
||||
per row, so its answer is read from texts, as it was before rows could be
|
||||
returned at all."""
|
||||
if original_rows is None or shown_rows is None or len(returned_rows) != len(original_rows):
|
||||
"""A row the server echoes back is restored to the original row object. A server
|
||||
that echoes every row back has not rewritten anything per row, so its answer is
|
||||
read from texts, as it was before rows could be returned at all."""
|
||||
if original_rows is None or compared_rows is None or len(returned_rows) != len(original_rows):
|
||||
return tuple(returned_rows)
|
||||
if all(returned == shown for shown, returned in zip(shown_rows, returned_rows)):
|
||||
echoed: Final = tuple(
|
||||
is_echo(compared, returned) for compared, returned in zip(compared_rows, returned_rows, strict=True)
|
||||
)
|
||||
if all(echoed):
|
||||
return None
|
||||
return tuple(
|
||||
original if returned == shown else returned
|
||||
for original, shown, returned in zip(original_rows, shown_rows, returned_rows)
|
||||
original if was_echoed else returned
|
||||
for original, returned, was_echoed in zip(original_rows, returned_rows, echoed, strict=True)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -200,6 +248,7 @@ class GenericGuardrailAPI(CustomGuardrail):
|
|||
additional_provider_specific_params: Mapping[str, object] | None = None,
|
||||
unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed",
|
||||
fail_on_error: bool | None = True,
|
||||
forward_messages_as_sent: bool | None = False,
|
||||
extra_headers: list | None = None,
|
||||
streaming_end_of_stream_only: bool | None = None,
|
||||
streaming_sampling_rate: int | None = None,
|
||||
|
|
@ -235,6 +284,7 @@ class GenericGuardrailAPI(CustomGuardrail):
|
|||
self.unreachable_fallback: Literal["fail_closed", "fail_open"] = unreachable_fallback
|
||||
|
||||
self.fail_on_error: bool = True if fail_on_error is None else fail_on_error
|
||||
self.forward_messages_as_sent: bool = bool(forward_messages_as_sent)
|
||||
|
||||
# Read by UnifiedLLMGuardrails.async_post_call_streaming_iterator_hook
|
||||
# via getattr(guardrail_to_apply, "streaming_*", default).
|
||||
|
|
@ -316,15 +366,17 @@ class GenericGuardrailAPI(CustomGuardrail):
|
|||
http_status_code: int | None = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
status_suffix: Final = f" http_status_code={http_status_code}" if http_status_code else ""
|
||||
error_type: Final = "HTTPStatusError" if isinstance(error, httpx.HTTPStatusError) else type(error).__name__
|
||||
verbose_proxy_logger.critical(
|
||||
"Generic Guardrail API error (fail-open). Proceeding without guardrail.%s "
|
||||
"guardrail_name=%s api_base=%s input_type=%s litellm_call_id=%s litellm_trace_id=%s",
|
||||
"guardrail_name=%s api_base=%s input_type=%s litellm_call_id=%s litellm_trace_id=%s error_type=%s",
|
||||
status_suffix,
|
||||
getattr(self, "guardrail_name", None),
|
||||
getattr(self, "api_base", None),
|
||||
input_type,
|
||||
getattr(logging_obj, "litellm_call_id", None) if logging_obj else None,
|
||||
getattr(logging_obj, "litellm_trace_id", None) if logging_obj else None,
|
||||
error_type,
|
||||
exc_info=error,
|
||||
)
|
||||
# Keep flow going - treat as action=NONE (no modifications)
|
||||
|
|
@ -339,10 +391,23 @@ class GenericGuardrailAPI(CustomGuardrail):
|
|||
headers.update(self.headers)
|
||||
return headers
|
||||
|
||||
def _request_body(
|
||||
self,
|
||||
guardrail_request: GenericGuardrailAPIRequest,
|
||||
structured_messages: Sequence[AllMessageValues] | None,
|
||||
) -> Mapping[str, object]:
|
||||
body: Final[Mapping[str, object]] = guardrail_request.model_dump(mode="json")
|
||||
if not self.forward_messages_as_sent or structured_messages is None:
|
||||
return body
|
||||
return {
|
||||
**body,
|
||||
"structured_messages": _SENT_ROWS.dump_python(_SENT_ROWS.validate_python(structured_messages), mode="json"),
|
||||
}
|
||||
|
||||
def _build_guardrail_return_inputs(
|
||||
self,
|
||||
*,
|
||||
texts: list,
|
||||
texts: list[str],
|
||||
images: list[str] | None,
|
||||
tools: list[ChatCompletionToolParam] | None,
|
||||
structured_messages: Sequence[AllMessageValues] | None,
|
||||
|
|
@ -361,8 +426,13 @@ class GenericGuardrailAPI(CustomGuardrail):
|
|||
return_inputs["tools"] = guardrail_response.tools
|
||||
elif tools:
|
||||
return_inputs["tools"] = tools
|
||||
compared_rows, is_echo = (
|
||||
(structured_messages, _is_echo) if self.forward_messages_as_sent else (shown_messages, _is_shown_row)
|
||||
)
|
||||
rows_to_write_back: Final = (
|
||||
_structured_rows_to_write_back(structured_messages, shown_messages, guardrail_response.structured_messages)
|
||||
_structured_rows_to_write_back(
|
||||
structured_messages, compared_rows, guardrail_response.structured_messages, is_echo
|
||||
)
|
||||
if guardrail_response.structured_messages
|
||||
else None
|
||||
)
|
||||
|
|
@ -465,7 +535,7 @@ class GenericGuardrailAPI(CustomGuardrail):
|
|||
litellm_version=litellm_version,
|
||||
images=images,
|
||||
tools=([GuardrailToolParam.model_validate(t) for t in tools] if tools else None),
|
||||
structured_messages=structured_messages,
|
||||
structured_messages=None if self.forward_messages_as_sent else structured_messages,
|
||||
tool_calls=tool_calls,
|
||||
additional_provider_specific_params=additional_params,
|
||||
input_type=input_type,
|
||||
|
|
@ -478,7 +548,7 @@ class GenericGuardrailAPI(CustomGuardrail):
|
|||
# Use mode="json" to ensure all iterables are converted to lists
|
||||
response: Final = await self.async_handler.post(
|
||||
url=self.api_base,
|
||||
json=guardrail_request.model_dump(mode="json"),
|
||||
json=dict(self._request_body(guardrail_request, structured_messages)),
|
||||
headers=headers,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,23 +6,23 @@ knowledge lives in one place. Uses guardrail translation handlers where availabl
|
|||
with standalone extractors for generate_content and MCP.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Final
|
||||
|
||||
from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
|
||||
from litellm.llms import load_guardrail_translation_mappings
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import list_field
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
# Call types that have no guardrail translation handler; we use standalone extractors
|
||||
STANDALONE_EXTRACTORS: Final[dict[str, Any]] = {}
|
||||
|
||||
|
||||
def _extract_generate_content_tool_names(data: dict) -> list[str]:
|
||||
def _extract_generate_content_tool_names(data: Mapping[str, object]) -> list[str]:
|
||||
"""Google generateContent: tools[].functionDeclarations[].name"""
|
||||
names: Final[list[str]] = []
|
||||
for tool in data.get("tools") or []:
|
||||
if not isinstance(tool, dict):
|
||||
continue
|
||||
for decl in tool.get("functionDeclarations") or []:
|
||||
for tool in list_field(data, "tools"):
|
||||
for decl in list_field(tool, "functionDeclarations"):
|
||||
if isinstance(decl, dict) and decl.get("name"):
|
||||
names.append(str(decl["name"]))
|
||||
return names
|
||||
|
|
|
|||
|
|
@ -425,7 +425,7 @@ async def add_new_user_to_default_team(
|
|||
user_id: str,
|
||||
user_email: str | None,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
teams: list[str] | list[NewUserRequestTeam],
|
||||
teams: Sequence[str | NewUserRequestTeam],
|
||||
prisma_client: "PrismaClient",
|
||||
):
|
||||
tasks: Final[list[Awaitable[object]]] = []
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError,
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.constants import LITELLM_PROXY_ADMIN_NAME
|
||||
from litellm.constants import LITELLM_PROXY_ADMIN_NAME, TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.ptu_pricing import (
|
||||
|
|
@ -1881,7 +1881,7 @@ async def delete_team_models(
|
|||
"""
|
||||
deleted_model_ids: Final[list[str]] = []
|
||||
transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db)
|
||||
async with transaction_client.tx() as tx:
|
||||
async with transaction_client.tx(timeout=datetime.timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS)) as tx:
|
||||
for team_id in team_ids:
|
||||
rows = await _get_team_deployments(team_id, prisma_client, table=tx.litellm_proxymodeltable)
|
||||
model_ids = [row.model_id for row in rows]
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import traceback
|
|||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from collections.abc import Set as AbstractSet
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
|
|
@ -42,6 +42,7 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.caching.redis_batch import active_request_redis_batches
|
||||
from litellm.constants import TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.proxy._types import (
|
||||
|
|
@ -4674,7 +4675,7 @@ async def delete_team(
|
|||
# or is still waiting on the lock, in which case its own re-read happens after this commits
|
||||
# and sees the row gone before it writes anything.
|
||||
delete_filter: Final[_TeamIdInFilter] = {"team_id": {"in": data.team_ids}}
|
||||
async with prisma_client.tx() as tx:
|
||||
async with prisma_client.tx(timeout=timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS)) as tx:
|
||||
for team_id in sorted(data.team_ids):
|
||||
await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id)
|
||||
await tx.litellm_teamtable.delete_many(where=delete_filter)
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ from litellm.proxy._types import (
|
|||
Member,
|
||||
NewTeamRequest,
|
||||
NewUserRequest,
|
||||
NewUserRequestTeam,
|
||||
NewUserResponse,
|
||||
ProxyErrorTypes,
|
||||
ProxyException,
|
||||
|
|
@ -118,7 +119,11 @@ from litellm.proxy.common_utils.html_forms.jwt_display_template import (
|
|||
from litellm.proxy.common_utils.html_forms.ui_login import build_ui_login_form
|
||||
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
from litellm.proxy.management_endpoints.internal_user_endpoints import new_user
|
||||
from litellm.proxy.management_endpoints.internal_user_endpoints import (
|
||||
add_new_user_to_default_team,
|
||||
check_if_default_team_set,
|
||||
new_user,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.sso import CustomMicrosoftSSO, LiteLLMGoogleSSO, create_generic_sso_provider
|
||||
from litellm.proxy.management_endpoints.sso.id_jag_assertion_capture import (
|
||||
id_jag_assertion_capture_gap,
|
||||
|
|
@ -2825,6 +2830,52 @@ async def insert_sso_user(
|
|||
return response
|
||||
|
||||
|
||||
DefaultTeamJoin: TypeAlias = Callable[
|
||||
[LiteLLM_UserTable, Sequence[str | NewUserRequestTeam], PrismaClient], Awaitable[None]
|
||||
]
|
||||
|
||||
|
||||
async def _join_default_teams_as_admin(
|
||||
user: LiteLLM_UserTable, teams: Sequence[str | NewUserRequestTeam], prisma_client: PrismaClient
|
||||
) -> None:
|
||||
await add_new_user_to_default_team(
|
||||
user_id=user.user_id,
|
||||
user_email=user.user_email,
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
teams=teams,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
|
||||
|
||||
def _default_teams_not_joined(user: LiteLLM_UserTable) -> tuple[str | NewUserRequestTeam, ...]:
|
||||
joined: Final = frozenset(user.teams)
|
||||
return tuple(
|
||||
team
|
||||
for team in check_if_default_team_set() or ()
|
||||
if (team if isinstance(team, str) else team.team_id) not in joined
|
||||
)
|
||||
|
||||
|
||||
async def _insert_sso_user_or_find_concurrent_insert(
|
||||
insert: Awaitable[NewUserResponse],
|
||||
user_id: str | None,
|
||||
prisma_client: PrismaClient,
|
||||
join_default_teams: DefaultTeamJoin = _join_default_teams_as_admin,
|
||||
) -> NewUserResponse | LiteLLM_UserTable:
|
||||
try:
|
||||
return await insert
|
||||
except Exception:
|
||||
users: Final = UserRepository(prisma_client, use_writer=True)
|
||||
concurrent_insert: Final = None if user_id is None else await users.find_by_id(user_id)
|
||||
if concurrent_insert is None:
|
||||
raise
|
||||
unjoined: Final = _default_teams_not_joined(concurrent_insert)
|
||||
if not unjoined:
|
||||
return concurrent_insert
|
||||
await join_default_teams(concurrent_insert, unjoined, prisma_client)
|
||||
return await users.find_by_id(concurrent_insert.user_id) or concurrent_insert
|
||||
|
||||
|
||||
@router.get(
|
||||
"/sso/get/ui_settings",
|
||||
tags=["experimental"],
|
||||
|
|
@ -3471,9 +3522,10 @@ class SSOAuthenticationHandler:
|
|||
else:
|
||||
verbose_proxy_logger.info("user not in DB, inserting user into LiteLLM DB")
|
||||
# user not in DB, insert User into LiteLLM DB
|
||||
user_info = await insert_sso_user(
|
||||
result_openid=result,
|
||||
user_defined_values=user_defined_values,
|
||||
user_info = await _insert_sso_user_or_find_concurrent_insert(
|
||||
insert=insert_sso_user(result_openid=result, user_defined_values=user_defined_values),
|
||||
user_id=None if user_defined_values is None else user_defined_values.get("user_id"),
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
return user_info
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -10,12 +10,14 @@ from pydantic import ConfigDict, TypeAdapter
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import ANTHROPIC_BATCHES_ROUTE
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_content_from_model_response,
|
||||
)
|
||||
from litellm.llms.anthropic import get_anthropic_config
|
||||
from litellm.llms.anthropic.common_utils import anthropic_error_frame_exception
|
||||
from litellm.llms.anthropic.pass_through.stream_assembly import (
|
||||
build_complete_streaming_response,
|
||||
build_complete_streaming_response_legacy,
|
||||
|
|
@ -254,6 +256,26 @@ class AnthropicPassthroughLoggingHandler:
|
|||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def log_error_frame_as_failure(
|
||||
attempt_logging_obj: LiteLLMLoggingObj,
|
||||
request_body: Mapping[str, object],
|
||||
all_chunks: Sequence[str | bytes],
|
||||
error_event: tuple[str, str, int],
|
||||
) -> None:
|
||||
"""A stream closed at the provider's `event: error` frame is one failed attempt of the request: it logs
|
||||
through the attempt-scoped copy its caller took at the close, so the usage it consumed never reaches
|
||||
the success log of the attempt the router opens in its place."""
|
||||
error_type, message, status_code = error_event
|
||||
await asyncify(AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure)(
|
||||
litellm_logging_obj=attempt_logging_obj, request_body=request_body, all_chunks=all_chunks
|
||||
)
|
||||
await attempt_logging_obj.dispatch_failure_handlers(
|
||||
anthropic_error_frame_exception(error_type, message, status_code, str(attempt_logging_obj.model)),
|
||||
"",
|
||||
prefer_async_handlers=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _cost_partial_stream_or_zero(
|
||||
partial_response: ModelResponse | TextCompletionResponse, model: str, logging_obj: LiteLLMLoggingObj
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ class PassThroughStreamingHandler:
|
|||
)
|
||||
|
||||
logging_scheduled = False
|
||||
stream_ended = False # rebind-ok: flipped once the upstream ran to its end, so the finally can tell a close-at-frame from a completed stream
|
||||
model_name: Final = PassThroughStreamingHandler._extract_model_for_cost_injection(
|
||||
request_body=request_body,
|
||||
url_route=url_route,
|
||||
|
|
@ -224,6 +225,7 @@ class PassThroughStreamingHandler:
|
|||
)
|
||||
if pending:
|
||||
yield pending
|
||||
stream_ended = True # rebind-ok: the finally reads it to spot a close-at-frame
|
||||
# Stream completed cleanly. When the proxy armed deferred
|
||||
# dispatch (post-call guardrails active), park the logging
|
||||
# coroutine on logging_obj instead of enqueueing now, so
|
||||
|
|
@ -267,13 +269,46 @@ class PassThroughStreamingHandler:
|
|||
# a success would double-log the same request.
|
||||
if not logging_scheduled and raw_bytes and response.status_code < 400:
|
||||
logging_scheduled = True
|
||||
superseded_error_event: Final = (
|
||||
None
|
||||
if stream_ended
|
||||
else PassThroughStreamingHandler._error_frame_closed_at(endpoint_type, raw_bytes)
|
||||
)
|
||||
try:
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=_build_logging_coroutine())
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
async_coroutine=(
|
||||
_build_logging_coroutine()
|
||||
if superseded_error_event is None
|
||||
else AnthropicPassthroughLoggingHandler.log_error_frame_as_failure(
|
||||
attempt_logging_obj=litellm_logging_obj.attempt_scoped_copy(),
|
||||
request_body=resolved_request_body,
|
||||
all_chunks=tuple(raw_bytes),
|
||||
error_event=superseded_error_event,
|
||||
)
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e)
|
||||
else:
|
||||
bind_budget_reservation_to_callbacks(litellm_logging_obj.litellm_params)
|
||||
|
||||
@staticmethod
|
||||
def _error_frame_closed_at(endpoint_type: EndpointType, raw_bytes: Sequence[bytes]) -> tuple[str, str, int] | None:
|
||||
"""The provider `event: error` frame of a stream the router closed because a retry or fallback took the
|
||||
request over. That attempt is logged as its own failure, on a copy taken now so the retry's rebinding of
|
||||
the shared logging object never reaches it. A stream that ran to its end or that the client closed keeps
|
||||
logging through the success route, so its usage is still billed."""
|
||||
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
|
||||
closing_superseded_attempt,
|
||||
parse_anthropic_error_event,
|
||||
)
|
||||
|
||||
if endpoint_type != EndpointType.ANTHROPIC or not closing_superseded_attempt():
|
||||
return None
|
||||
|
||||
complete_frames, _ = split_complete_sse_frames(b"".join(raw_bytes))
|
||||
return parse_anthropic_error_event(complete_frames)
|
||||
|
||||
@staticmethod
|
||||
async def route_streaming_logging_to_handler(
|
||||
litellm_logging_obj: LiteLLMLoggingObj,
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from litellm.integrations.custom_guardrail import ModifyResponseException
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.litellm_core_utils.tool_fields import first_validation_error
|
||||
from litellm.llms.openai.responses.guardrail_translation.handler import (
|
||||
OpenAIResponsesHandler,
|
||||
build_blocked_response,
|
||||
|
|
@ -45,7 +46,12 @@ from litellm.proxy.auth.user_api_key_auth import (
|
|||
user_api_key_auth,
|
||||
user_api_key_auth_websocket,
|
||||
)
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing, cost_headers, create_response
|
||||
from litellm.proxy.common_request_processing import (
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
cost_headers,
|
||||
create_response,
|
||||
resolve_counted_tool_fields,
|
||||
)
|
||||
from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports
|
||||
_read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
_safe_set_request_parsed_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
|
|
@ -57,6 +63,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio
|
|||
request_tags_from_request_data,
|
||||
)
|
||||
from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
|
||||
from litellm.router import Router
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import (
|
||||
REASONING_EFFORT,
|
||||
|
|
@ -69,7 +76,6 @@ from litellm.types.utils import TokenCountResponse
|
|||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = APIRouter()
|
||||
_RESPONSES_WS_CONFIG_VALUE_ADAPTER: Final[TypeAdapter[object | None]] = TypeAdapter(object | None)
|
||||
|
|
@ -1376,6 +1382,15 @@ def _proxy_token_counter() -> _TokenCounter:
|
|||
_token_counter_dep: Final = Depends(_proxy_token_counter)
|
||||
|
||||
|
||||
def _proxy_llm_router() -> Router | None:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
return llm_router
|
||||
|
||||
|
||||
_llm_router_dep: Final = Depends(_proxy_llm_router)
|
||||
|
||||
|
||||
def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse:
|
||||
body: Final[_ResponsesApiErrorBody] = {
|
||||
"error": {
|
||||
|
|
@ -1432,6 +1447,8 @@ def _responses_input_as_token_count_messages(
|
|||
async def responses_input_tokens(
|
||||
request: Request,
|
||||
token_counter: _TokenCounter = _token_counter_dep,
|
||||
llm_router: Router | None = _llm_router_dep,
|
||||
user_api_key_dict: UserAPIKeyAuth = _user_api_key_auth_dep,
|
||||
):
|
||||
"""
|
||||
Count the input tokens of a Responses API request without calling the model.
|
||||
|
|
@ -1464,6 +1481,13 @@ async def responses_input_tokens(
|
|||
code="missing_required_parameter",
|
||||
)
|
||||
|
||||
counted_fields: Final = {
|
||||
name: data[name] for name in ("model", "tools", "drop_params", "additional_drop_params") if name in data
|
||||
}
|
||||
token_fields: Final = resolve_counted_tool_fields(counted_fields, llm_router, user_api_key_dict.team_id)
|
||||
if isinstance(token_fields, ProxyException):
|
||||
return _responses_invalid_request_response(message=token_fields.message, param=token_fields.param, code=None)
|
||||
|
||||
try:
|
||||
payload: Final[_TokenCountPayload] = {
|
||||
"model": model_name,
|
||||
|
|
@ -1471,9 +1495,13 @@ async def responses_input_tokens(
|
|||
input_value=input_value,
|
||||
instructions=data.get("instructions"),
|
||||
),
|
||||
"tools": data.get("tools"),
|
||||
"tools": token_fields.get("tools"),
|
||||
}
|
||||
token_request: Final = TokenCountRequest.model_validate(payload)
|
||||
except ValidationError as e:
|
||||
return _responses_invalid_request_response(
|
||||
message=first_validation_error(e), param=str(e.errors(include_url=False)[0]["loc"][0]), code=None
|
||||
)
|
||||
except Exception as e:
|
||||
return _responses_invalid_request_response(
|
||||
message=f"Invalid request for token counting: {e}", param=None, code=None
|
||||
|
|
|
|||
|
|
@ -3493,6 +3493,7 @@ class ProxyLogging:
|
|||
Is triggered when self._is_proxy_only_error() returns True
|
||||
"""
|
||||
litellm_logging_obj: Logging | None = request_data.get("litellm_logging_obj", None)
|
||||
set_up_by_request: Final = litellm_logging_obj is not None
|
||||
if litellm_logging_obj is None:
|
||||
from litellm._uuid import uuid
|
||||
|
||||
|
|
@ -3557,7 +3558,9 @@ class ProxyLogging:
|
|||
input = request_data["input"]
|
||||
litellm_logging_obj.model_call_details["input"] = input
|
||||
body_shape_call_type = CallTypes.aembedding.value
|
||||
resolved_call_type: Final = _call_type_for_route(route) or body_shape_call_type
|
||||
resolved_call_type: Final = _call_type_for_route(route) or (
|
||||
None if set_up_by_request else body_shape_call_type
|
||||
)
|
||||
if resolved_call_type is not None and litellm_logging_obj.call_type != CallTypes.pass_through.value:
|
||||
litellm_logging_obj.call_type = resolved_call_type
|
||||
litellm_logging_obj.model_call_details["call_type"] = resolved_call_type
|
||||
|
|
|
|||
|
|
@ -123,6 +123,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
|
|||
mask_sensitive_structure,
|
||||
)
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, anthropic_error_frame_exception
|
||||
from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
|
||||
from litellm.llms.base_llm.vector_store.transformation import (
|
||||
|
|
@ -537,6 +538,7 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream])
|
|||
|
||||
|
||||
_NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({})
|
||||
_NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
|
@ -573,6 +575,21 @@ def _deployment_that_just_failed(request_metadata: object) -> str | None:
|
|||
return model_id if isinstance(model_id, str) else None
|
||||
|
||||
|
||||
def _drops_malformed_request_tools(
|
||||
deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object]
|
||||
) -> bool:
|
||||
if isinstance(request.get("tools") or [], (list, tuple)) or not deployment_params.get("tools"):
|
||||
return False
|
||||
set_defaults: Final = {name: value for name, value in defaults.items() if value is not None}
|
||||
return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools")
|
||||
|
||||
|
||||
def _merged_tools(deployment_tools: Sequence[object], request_tools: object) -> list[object] | None:
|
||||
if not isinstance(request_tools, (list, tuple)):
|
||||
return None
|
||||
return [*deployment_tools, *request_tools] if deployment_tools or request_tools else None
|
||||
|
||||
|
||||
def _deployment_pick_attributes(model: str, request_kwargs: Mapping[str, object] | None) -> Mapping[str, str | int]:
|
||||
"""Bounded attributes for one deployment pick; attempt is 1-based within the current model group."""
|
||||
kwargs: Final = request_kwargs or {}
|
||||
|
|
@ -4507,7 +4524,9 @@ class Router:
|
|||
return deployment_pydantic_obj
|
||||
|
||||
@staticmethod
|
||||
def _merge_tools_from_deployment(deployment: dict, kwargs: dict) -> None:
|
||||
def _merge_tools_from_deployment(
|
||||
deployment: dict, kwargs: dict, defaults: Mapping[str, object] = _NO_DEFAULT_PARAMS
|
||||
) -> None:
|
||||
"""
|
||||
Merge tools from deployment litellm_params with request kwargs.
|
||||
When both have tools, concatenate them (deployment tools first, then request tools).
|
||||
|
|
@ -4518,11 +4537,12 @@ class Router:
|
|||
dep_params = dep_params_raw
|
||||
else:
|
||||
dep_params = dep_params_raw.model_dump(exclude_none=True)
|
||||
dep_tools: Final = dep_params.get("tools") or []
|
||||
req_tools: Final = kwargs.get("tools") or []
|
||||
if dep_tools or req_tools:
|
||||
merged: Final = list(dep_tools) + list(req_tools)
|
||||
kwargs["tools"] = merged
|
||||
if _drops_malformed_request_tools(dep_params, defaults, kwargs):
|
||||
for name in TOOL_FIELDS:
|
||||
kwargs.pop(name, None)
|
||||
merged_tools: Final = _merged_tools(dep_params.get("tools") or [], kwargs.get("tools") or [])
|
||||
if merged_tools is not None:
|
||||
kwargs["tools"] = merged_tools
|
||||
if "tool_choice" not in kwargs and dep_params.get("tool_choice") is not None:
|
||||
kwargs["tool_choice"] = dep_params["tool_choice"]
|
||||
|
||||
|
|
@ -4543,7 +4563,7 @@ class Router:
|
|||
deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ())
|
||||
):
|
||||
kwargs.pop(key, None)
|
||||
self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs)
|
||||
self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=self.default_litellm_params)
|
||||
|
||||
model_info = deployment.get("model_info", {}).copy()
|
||||
deployment_litellm_model_name = deployment["litellm_params"]["model"]
|
||||
|
|
@ -6035,6 +6055,7 @@ class Router:
|
|||
"""
|
||||
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
|
||||
aclose_if_supported,
|
||||
aclose_superseded_attempt,
|
||||
parse_anthropic_error_event,
|
||||
parse_anthropic_refusal_stop_details,
|
||||
)
|
||||
|
|
@ -6044,6 +6065,14 @@ class Router:
|
|||
|
||||
source_iterator: Final = response
|
||||
|
||||
async def close_source() -> None:
|
||||
with anyio.CancelScope(shield=True), contextlib.suppress(BaseException):
|
||||
await aclose_if_supported(source_iterator)
|
||||
|
||||
async def close_superseded() -> None:
|
||||
with anyio.CancelScope(shield=True), contextlib.suppress(BaseException):
|
||||
await aclose_superseded_attempt(source_iterator)
|
||||
|
||||
async def stream_with_fallbacks() -> AsyncGenerator[bytes, None]:
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
||||
|
|
@ -6130,11 +6159,11 @@ class Router:
|
|||
model,
|
||||
initial_kwargs,
|
||||
wrapper,
|
||||
close_superseded,
|
||||
):
|
||||
yield item
|
||||
finally:
|
||||
with anyio.CancelScope(shield=True), contextlib.suppress(BaseException):
|
||||
await aclose_if_supported(source_iterator)
|
||||
await close_source()
|
||||
|
||||
# Referenced by stream_with_fallbacks via closure - assigned here, before
|
||||
# the generator body ever runs, so the reference resolves fine despite
|
||||
|
|
@ -6150,6 +6179,7 @@ class Router:
|
|||
model: str,
|
||||
initial_kwargs: dict[str, Any], # mutable-ok: handed to _aanthropic_messages_fallback_attempt, which mutates it
|
||||
wrapper: "FallbackAwareAnthropicMessagesStream",
|
||||
close_superseded: Callable[[], Awaitable[None]],
|
||||
) -> AsyncGenerator[bytes, None]:
|
||||
"""Turns a source-iterator failure into a fallback attempt or the error reaching the caller."""
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
|
|
@ -6169,6 +6199,7 @@ class Router:
|
|||
)
|
||||
if fallback_error is None:
|
||||
raise stream_error
|
||||
await close_superseded()
|
||||
outcome: Final = await self._astream_retry_same_group(
|
||||
fallback_error,
|
||||
initial_kwargs,
|
||||
|
|
|
|||
|
|
@ -364,10 +364,10 @@ def filter_web_search_deployments(
|
|||
if isinstance(healthy_deployments, dict):
|
||||
return healthy_deployments
|
||||
|
||||
tools_value: Final[object] = request_kwargs.get("tools") or []
|
||||
tools: Final = _OBJECT_TUPLE_ADAPTER.validate_python(tools_value)
|
||||
is_web_search_request: Final = any(_is_web_search_tool(tool) for tool in tools)
|
||||
|
||||
tools: Final[object] = request_kwargs.get("tools")
|
||||
is_web_search_request: Final = isinstance(tools, (list, tuple)) and any(
|
||||
_is_web_search_tool(tool) for tool in _OBJECT_TUPLE_ADAPTER.validate_python(tools)
|
||||
)
|
||||
if not is_web_search_request:
|
||||
return healthy_deployments
|
||||
|
||||
|
|
|
|||
|
|
@ -329,6 +329,12 @@ def _should_run_cooldown_logic(
|
|||
)
|
||||
return False
|
||||
|
||||
if isinstance(original_exception, litellm.MalformedRequestFieldError):
|
||||
verbose_router_logger.debug(
|
||||
"Should Not Run Cooldown Logic: litellm rejected the request before calling the deployment"
|
||||
)
|
||||
return False
|
||||
|
||||
if request_kwargs is not None and _is_per_user_provider_request(request_kwargs):
|
||||
verbose_router_logger.debug(
|
||||
"Should Not Run Cooldown Logic: the failure came from one caller's per-user provider session"
|
||||
|
|
|
|||
|
|
@ -1195,6 +1195,16 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up
|
|||
description="Maximum decisions calls in flight at once on the guardrail in each proxy worker. Implemented by guardrail='decision_model'.",
|
||||
)
|
||||
|
||||
forward_messages_as_sent: bool | None = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the "
|
||||
"guardrail exactly as the client sent it, so rows can carry provider-native content blocks and "
|
||||
"keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with "
|
||||
"undeclared keys removed."
|
||||
),
|
||||
)
|
||||
|
||||
extra_headers: list[str] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Final, Literal, cast # noqa: TID251 # JSON chat rows have no typed constructor across roles
|
||||
|
||||
from pydantic import ConfigDict, Field
|
||||
from pydantic import ConfigDict, Field, SerializerFunctionWrapHandler, model_serializer
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
|
@ -17,12 +17,20 @@ class GuardrailToolParam(LiteLLMBaseModel):
|
|||
"""A tool forwarded verbatim to the guardrail for inspection.
|
||||
|
||||
Built-in tools (code_interpreter, file_search, ...) have no ``function`` block
|
||||
and stash their config in tool-specific keys, so only ``type`` is required and
|
||||
``extra="allow"`` preserves the rest instead of stripping it.
|
||||
and stash their config in tool-specific keys, and provider-native tools such as
|
||||
Gemini ``function_declarations`` carry no ``type`` at all, so nothing is required
|
||||
and ``extra="allow"`` preserves the rest instead of stripping it.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
type: str
|
||||
type: str | None = None
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _without_absent_type(self, handler: SerializerFunctionWrapHandler) -> Mapping[str, object]:
|
||||
dumped: Final = cast("Mapping[str, object]", handler(self)) # cast-ok: handler dumps this model's fields
|
||||
if self.type is not None:
|
||||
return dumped
|
||||
return {key: value for key, value in dumped.items() if key != "type"}
|
||||
|
||||
|
||||
class GenericGuardrailAPIMetadata(TypedDict, total=False):
|
||||
|
|
@ -62,6 +70,15 @@ class GenericGuardrailAPIOptionalParams(LiteLLMBaseModel):
|
|||
),
|
||||
)
|
||||
|
||||
forward_messages_as_sent: bool | None = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can "
|
||||
"carry provider-native content blocks and keys the OpenAI chat types do not declare. False "
|
||||
"(default) sends OpenAI-format chat rows with undeclared keys removed."
|
||||
),
|
||||
)
|
||||
|
||||
streaming_end_of_stream_only: bool | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
|
|
|
|||
|
|
@ -92,6 +92,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
|
|||
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
|
||||
from litellm.litellm_core_utils.tokenizer import strip_special_tokens
|
||||
from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields
|
||||
from litellm.rust_bridge import tokenizer as tokenizer_dispatch
|
||||
from litellm.rust_bridge.catalog import decision
|
||||
from litellm.rust_bridge.configuration import Decision
|
||||
|
|
@ -474,6 +475,7 @@ from .exceptions import (
|
|||
BudgetExceededError,
|
||||
ContentPolicyViolationError,
|
||||
ContextWindowExceededError,
|
||||
MalformedRequestFieldError,
|
||||
ModelNotMappedError,
|
||||
NotFoundError,
|
||||
OpenAIError,
|
||||
|
|
@ -1495,7 +1497,23 @@ def check_coroutine(value) -> bool:
|
|||
return get_coroutine_checker().is_async_callable(value)
|
||||
|
||||
|
||||
async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str):
|
||||
def _deployment_provider(kwargs: Mapping[str, object]) -> str:
|
||||
model: Final = kwargs.get("model")
|
||||
custom_llm_provider: Final = kwargs.get("custom_llm_provider")
|
||||
api_base: Final = kwargs.get("api_base")
|
||||
if not isinstance(model, str):
|
||||
return ""
|
||||
try:
|
||||
return litellm.get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
|
||||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
)[1]
|
||||
except BadRequestError:
|
||||
return ""
|
||||
|
||||
|
||||
async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str) -> dict[str, Any]:
|
||||
"""
|
||||
Allow modifying the request just before it's sent to the deployment.
|
||||
|
||||
|
|
@ -1506,7 +1524,16 @@ async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str)
|
|||
except ValueError:
|
||||
typed_call_type = None # unknown call type
|
||||
|
||||
modified_kwargs = kwargs.copy()
|
||||
route: Final[object] = getattr(kwargs.get("litellm_logging_obj"), "call_type", None)
|
||||
resolved: Final = resolve_deployment_tool_fields(kwargs, route)
|
||||
if isinstance(resolved, MalformedField):
|
||||
raise MalformedRequestFieldError(
|
||||
message=resolved.message,
|
||||
model=kwargs.get("model"),
|
||||
llm_provider=_deployment_provider(kwargs),
|
||||
body={"type": "invalid_request_error", "param": resolved.field},
|
||||
)
|
||||
modified_kwargs = dict(resolved)
|
||||
|
||||
custom_logger_class: Final = _get_cached_custom_logger()
|
||||
for callback in litellm.callbacks:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# litellm/proxy/vector_stores/vector_store_registry.py
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from itertools import chain
|
||||
from typing import (
|
||||
|
|
@ -11,7 +11,7 @@ from typing import (
|
|||
get_args,
|
||||
)
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import remove_items_at_indices
|
||||
|
|
@ -34,6 +34,26 @@ else:
|
|||
PrismaClient = Any
|
||||
|
||||
_DB_ROW: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
|
||||
_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
|
||||
_TOOL: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(strict=True))
|
||||
_STORE_ID_ITEMS: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
def _named_store_ids(store_ids: object) -> tuple[str, ...]:
|
||||
if isinstance(store_ids, str):
|
||||
return (store_ids,)
|
||||
try:
|
||||
items: Final = _STORE_ID_ITEMS.validate_python(store_ids)
|
||||
except ValidationError:
|
||||
return ()
|
||||
return tuple(store_id for store_id in items if isinstance(store_id, str))
|
||||
|
||||
|
||||
def _tool_vector_store_ids(tool: object) -> tuple[str, ...]:
|
||||
try:
|
||||
return _named_store_ids(_TOOL.validate_python(tool).get("vector_store_ids"))
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
class VectorStoreIndexRegistry:
|
||||
|
|
@ -131,7 +151,7 @@ class VectorStoreRegistry:
|
|||
|
||||
return VectorStoreToolParams(**kwargs)
|
||||
|
||||
def get_vector_store_ids_to_run(self, non_default_params: dict, tools: list[dict] | None = None) -> list[str]:
|
||||
def get_vector_store_ids_to_run(self, non_default_params: dict, tools: object = None) -> list[str]:
|
||||
"""
|
||||
Returns the vector store ids to run
|
||||
|
||||
|
|
@ -175,7 +195,7 @@ class VectorStoreRegistry:
|
|||
tools_to_remove: Final[list[int]] = []
|
||||
|
||||
for i, tool in enumerate(tools):
|
||||
tool_vector_store_ids = tool.get("vector_store_ids", [])
|
||||
tool_vector_store_ids = _tool_vector_store_ids(tool)
|
||||
if not tool_vector_store_ids:
|
||||
continue
|
||||
|
||||
|
|
@ -395,14 +415,15 @@ class VectorStoreRegistry:
|
|||
|
||||
return vector_stores_to_run
|
||||
|
||||
def _get_vector_store_ids_from_tool_calls(self, tools: list[dict] | None, vector_store_ids: list[str]) -> list[str]:
|
||||
def _get_vector_store_ids_from_tool_calls(self, tools: object, vector_store_ids: list[str]) -> list[str]:
|
||||
"""
|
||||
Returns the vector store ids from the tool calls
|
||||
"""
|
||||
return [
|
||||
*vector_store_ids,
|
||||
*chain.from_iterable(tool["vector_store_ids"] for tool in tools or () if "vector_store_ids" in tool),
|
||||
]
|
||||
try:
|
||||
validated_tools: Final = _ITEMS.validate_python(tools)
|
||||
except ValidationError:
|
||||
return vector_store_ids
|
||||
return [*vector_store_ids, *chain.from_iterable(_tool_vector_store_ids(tool) for tool in validated_tools)]
|
||||
|
||||
def load_vector_stores_from_config(self, vector_stores_config: list[dict]):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -112,6 +112,8 @@ ignored_function_names = [
|
|||
"_get_wildcard_deployments", # Tested through the get_model_list_of_routed_group wildcard test in test_router.py
|
||||
"_is_fallback_hop", # Tested through the order fallback hop tests in test_router_order_fallback.py
|
||||
"_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py
|
||||
"_merged_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
|
||||
"_drops_malformed_request_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
|
||||
"_collect_ptu_share_teams", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
|
||||
"_resolve_team_ptu_ceiling", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
|
||||
"_register_continuation_prefill_check", # Tested through the continuation tests in test_router.py
|
||||
|
|
|
|||
|
|
@ -25,3 +25,15 @@ By the domain a user would name: `pricing`, `spend`, `routing`, `mcp`. A file on
|
|||
directory that a `GROUPS` entry in `run.py` selects. There is no manifest and no `covers` marker on new
|
||||
tests. A product bug the test exposes is `pytest.skip("BUG: <symptom>")` at the top of the body, not a
|
||||
fix in the test and not a deletion. Needs no network, proxy, DB or Redis: `tests/unit`
|
||||
|
||||
CircleCI's integration groups run every file here except the OCI and sandbox tests, which no CI runs, and
|
||||
the files in `GITHUB_FILES` in `run.py`. CircleCI runs on main, not on pull requests, so a new file here
|
||||
first runs after its PR merges. Wanting a file to run before merge is no reason to change CI, and a new
|
||||
dedicated workflow for it is the same mistake as a shared shard. Write the file, leave `GITHUB_FILES` and
|
||||
every workflow alone, and tell the user it first runs on main. `GITHUB_FILES` is only for a file that
|
||||
needs what CircleCI's groups cannot give it, like the images `image-scan.yml` builds or the plain pytest
|
||||
run CircleCI's `roi-database` job gives `test_roi_observed.py`. Add a file there only in that case,
|
||||
together with its dedicated job, and list a new dedicated GitHub workflow with the directory it owns in
|
||||
`DEDICATED_INTEGRATION_WORKFLOWS` in `.github/scripts/assert_ci_coverage.py`. The `ci-coverage` check
|
||||
fails a file here that a GitHub workflow runs unless it is in `GITHUB_FILES` and that workflow owns its
|
||||
directory (#46780)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local
|
|||
|
||||
The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport
|
||||
|
||||
The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so they run in the GitHub Actions `redis-compat` matrix against every supported redis-py major rather than in `run_integration.sh`
|
||||
The Redis cluster IAM contracts in `sdk/test_redis_cluster_iam_auth.py` start their own `redis-server` and mock the cloud credential clients, so CircleCI's `sdk` group installs `redis-server` before it runs them
|
||||
|
||||
The `cost` group is driven by the `CostTrackingTestCase` literals under `cost_calculation/cases/<provider>.py` (gathered by `cost_calculation/catalog.py`), with the scripted cost map in `cost_calculation/cost_map.py`. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/<scenario_id>`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL`
|
||||
|
||||
|
|
@ -26,7 +26,7 @@ Reuse the existing canned provider handlers through `_support/upstream.py`. It r
|
|||
|
||||
Owned proxies disable `.env` loading by default. A contract testing dotenv behavior can explicitly set `PYTHON_DOTENV_DISABLED=0` in its owned environment. The runner drops inherited `*_API_KEY` variables, so a contract that needs one sets a synthetic value in its owned environment. An explicit scratch writer database drops an inherited reader with a different database name; pass both database URLs when the contract needs a paired reader. Database relays close listeners, active connections and their event loops on teardown. Upstream startup failures reap the child process before returning the failure. Custom subprocesses, direct SQL and background tasks still need an explicit owner in the test
|
||||
|
||||
Owned proxy workers configure a 32-thread executor before application startup, preserving additional worker startup hooks. Held synchronous provider calls must not consume a CPU-dependent executor limit before the rest of a fault burst can start. The proxy entrypoint imports LiteLLM inside `main`, so spawned workers begin supervision before importing the application. Worker healthchecks have a five-second deadline by default, so uvicorn kills a worker that stays paused longer than that. Tests never signal proxy processes themselves. `OwnedProxy.kill_worker` SIGKILLs a worker and returns once its replacement answers readiness with its own pid, `replacing_worker` does the same for a kill that has to land mid-burst, and `suspended` pauses a worker or the supervisor, which only works on a proxy launched with `healthcheck=WorkerHealthcheck.LENIENT`. Helper processes that are not the proxy stop through `stop_child`, `stop_processes` and `stop_process_group` and pause through `paused`. `scripts/check_test_quality.py` rule TQ010 rejects a raw `os.kill`, SIGKILL / SIGSTOP / SIGCONT, `.kill()`, `.terminate()`, `.suspend()` or `.send_signal()` anywhere under `tests/integration` outside `_support`, and TQ011 rejects an assertion that caps an elapsed time with a fixed number. Bound timings with `_support/timing.py` instead, relative to a measured control or to the injected delay. `run.py` measures how long this box takes to import the proxy application before it starts pytest, records it in `boot-calibration.json` beside the results and exports it as `INTEGRATION_IMPORT_SECONDS`. Readiness deadlines, graceful stops and the default per-test timeout are multiples of that measurement (`_support/budget.py`), so a slower box gets longer budgets and a two-worker proxy gets twice the readiness deadline of a one-worker one. Readiness requires responsive worker identities that are still alive and a positively identified owned listener. Unreadable socket records cannot establish ownership. Crash tests own their client tasks with `TaskGroup`, release held peers in `finally` and verify that the intended worker dies. A test requiring traffic on both workers prepares that traffic explicitly instead of assuming OS load balancing, for example by sending each half while the other worker is suspended
|
||||
Owned proxy workers configure a 32-thread executor before application startup, preserving additional worker startup hooks. Held synchronous provider calls must not consume a CPU-dependent executor limit before the rest of a fault burst can start. The proxy entrypoint imports LiteLLM inside `main`, so spawned workers begin supervision before importing the application. Worker healthchecks have a five-second deadline by default, so uvicorn kills a worker that stays paused longer than that. Tests never signal proxy processes themselves. `OwnedProxy.kill_worker` SIGKILLs a worker and returns once its replacement answers readiness with its own pid, `replacing_worker` does the same for a kill that has to land mid-burst, and `suspended` pauses a worker or the supervisor, which only works on a proxy launched with `healthcheck=WorkerHealthcheck.LENIENT`. The `kill()` that `replacing_worker` yields returns as soon as the worker is dead, so a test can release requests it holds on the surviving worker right away. Uvicorn may take a full worker healthcheck to notice the death, so the block waits for the supervisor to reap the worker and for the replacement only when it exits. Helper processes that are not the proxy stop through `stop_child`, `stop_processes` and `stop_process_group` and pause through `paused`. `scripts/check_test_quality.py` rule TQ010 rejects a raw `os.kill`, SIGKILL / SIGSTOP / SIGCONT, `.kill()`, `.terminate()`, `.suspend()` or `.send_signal()` anywhere under `tests/integration` outside `_support`, and TQ011 rejects an assertion that caps an elapsed time with a fixed number. Bound timings with `_support/timing.py` instead, relative to a measured control or to the injected delay. `run.py` measures how long this box takes to import the proxy application before it starts pytest, records it in `boot-calibration.json` beside the results and exports it as `INTEGRATION_IMPORT_SECONDS`. Readiness deadlines, graceful stops and the default per-test timeout are multiples of that measurement (`_support/budget.py`), so a slower box gets longer budgets and a two-worker proxy gets twice the readiness deadline of a one-worker one. Readiness requires responsive worker identities that are still alive and a positively identified owned listener. Unreadable socket records cannot establish ownership. Crash tests own their client tasks with `TaskGroup`, release held peers in `finally` and verify that the intended worker dies. A test requiring traffic on both workers prepares that traffic explicitly instead of assuming OS load balancing, for example by sending each half while the other worker is suspended
|
||||
|
||||
Any port the harness picks before the server that listens on it has started comes from `allocate_port()` in `_support/ports.py`. That covers owned proxies, scripted upstreams, fake providers, OTLP sinks, database relays and Redis. A server that binds port 0 itself and keeps that socket, such as `wire_server` without a port, needs no allocation. Each pytest process claims a block of 200 ports between 10000 and 32767 by holding an exclusive lock file per block in the system temp directory, so two runs on one machine, such as parallel worktrees, never share a block, and the OS drops the lock when the process exits. The range sits above the fixed service ports and below the Linux and macOS ephemeral ranges, so a port-0 bind or an outbound connection never lands in a block. Ports inside a block go out in order and are never handed out twice in a run, a port that something outside the harness already holds is skipped, and a process that uses up its block claims the next free one. A test that restarts a server on the same port allocates that port up front instead of reading it back from the first server. TQ012 in `scripts/check_test_quality.py` rejects reading a port back from a port-0 bind anywhere under `tests/integration`
|
||||
|
||||
|
|
|
|||
|
|
@ -150,9 +150,9 @@ def _is_reaped(process: psutil.Process) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _wait_until_reaped(process: psutil.Process, supervisor: psutil.Process, seconds: float) -> None:
|
||||
def _wait_until_reaped(process: psutil.Process, seconds: float) -> None:
|
||||
deadline: Final = time.monotonic() + seconds
|
||||
while not _is_reaped(process) and not _is_stopped(supervisor):
|
||||
while not _is_reaped(process):
|
||||
assert time.monotonic() < deadline, f"The supervisor did not reap worker {process.pid} within {seconds:.0f}s"
|
||||
time.sleep(0.05)
|
||||
|
||||
|
|
@ -160,8 +160,6 @@ def _wait_until_reaped(process: psutil.Process, supervisor: psutil.Process, seco
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class WorkerKill:
|
||||
victim: psutil.Process
|
||||
supervisor: psutil.Process
|
||||
reap_seconds: float
|
||||
fired: threading.Event
|
||||
|
||||
def freeze(self) -> psutil.Process:
|
||||
|
|
@ -173,7 +171,6 @@ class WorkerKill:
|
|||
with suppress(psutil.NoSuchProcess):
|
||||
self.victim.kill()
|
||||
_wait_until_gone(self.victim, STRICT_HEALTHCHECK_SECONDS)
|
||||
_wait_until_reaped(self.victim, self.supervisor, self.reap_seconds)
|
||||
self.fired.set()
|
||||
|
||||
|
||||
|
|
@ -250,22 +247,22 @@ class OwnedProxy:
|
|||
assert pid in before, f"{pid} is not a serving worker of this proxy: {sorted(before)}"
|
||||
return psutil.Process(pid), before
|
||||
|
||||
def _worker_kill(self, victim: psutil.Process) -> WorkerKill:
|
||||
reap_seconds: Final = healthcheck_seconds(self.healthcheck) + STRICT_HEALTHCHECK_SECONDS
|
||||
return WorkerKill(victim, psutil.Process(self.process.pid), reap_seconds, threading.Event())
|
||||
def _replacement_of(self, victim: psutil.Process, before: frozenset[int]) -> int:
|
||||
_wait_until_reaped(victim, healthcheck_seconds(self.healthcheck) + STRICT_HEALTHCHECK_SECONDS)
|
||||
return self._await_replacement(before)
|
||||
|
||||
@contextmanager
|
||||
def replacing_worker(self, pid: int) -> Iterator[WorkerKill]:
|
||||
victim, before = self._victim(pid)
|
||||
kill: Final = self._worker_kill(victim)
|
||||
kill: Final = WorkerKill(victim, threading.Event())
|
||||
yield kill
|
||||
assert kill.fired.is_set(), f"replacing_worker({pid}) exited without killing the worker"
|
||||
self._await_replacement(before)
|
||||
self._replacement_of(victim, before)
|
||||
|
||||
def kill_worker(self, pid: int) -> int:
|
||||
victim, before = self._victim(pid)
|
||||
self._worker_kill(victim)()
|
||||
return self._await_replacement(before)
|
||||
WorkerKill(victim, threading.Event())()
|
||||
return self._replacement_of(victim, before)
|
||||
|
||||
@contextmanager
|
||||
def suspended(self, pid: int) -> Iterator[psutil.Process]:
|
||||
|
|
|
|||
|
|
@ -53,6 +53,13 @@ SELECT encode(sha256(convert_to(%s || '-' || n::text, 'UTF8')), 'hex'), %s, '{}'
|
|||
FROM generate_series(1, %s::int) AS n
|
||||
"""
|
||||
TAKE_TEAM_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext(%s))"
|
||||
LOCK_MODEL_ROW_SQL: Final = 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s FOR UPDATE'
|
||||
LOCK_HELD_PAST_DEFAULT_TRANSACTION_SECONDS: Final = 7
|
||||
OLDEST_TRANSACTION_BLOCKED_BY_SQL: Final = """
|
||||
SELECT coalesce(max(extract(epoch FROM clock_timestamp() - xact_start)), 0)::float AS age
|
||||
FROM pg_stat_activity
|
||||
WHERE %s::int = ANY(pg_blocking_pids(pid))
|
||||
"""
|
||||
# Sessions blocked on the advisory lock the given backend holds, and nothing else on the shared rig.
|
||||
WAITERS_ON_HELD_LOCK_SQL: Final = """
|
||||
SELECT count(*)::int AS waiting
|
||||
|
|
@ -131,6 +138,13 @@ def _waiters_on_lock_held_by(backend_pid: int) -> int:
|
|||
return waiting
|
||||
|
||||
|
||||
def _oldest_transaction_blocked_by(backend_pid: int) -> float:
|
||||
rows: Final = read_rows(OLDEST_TRANSACTION_BLOCKED_BY_SQL, (str(backend_pid),))
|
||||
age: Final = rows[0]["age"]
|
||||
assert isinstance(age, float)
|
||||
return age
|
||||
|
||||
|
||||
def _remove_team_by_sql(team_id: str) -> None:
|
||||
"""Cleanup for a team the test expects to have deleted itself. Whatever a failed delete left behind
|
||||
(row, memberships, `teams` references) goes by SQL so the shared rig stays clean without sending
|
||||
|
|
@ -161,6 +175,38 @@ def _create_team(scenario: Scenario, **fields: JsonValue) -> str:
|
|||
return team_id
|
||||
|
||||
|
||||
def _create_team_model(scenario: Scenario, team_id: str) -> str:
|
||||
created: Final = scenario.gateway.post(
|
||||
"/model/new",
|
||||
{
|
||||
"model_name": f"integration-{uuid.uuid4().hex}",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-integration"},
|
||||
"model_info": {"team_id": team_id},
|
||||
},
|
||||
)
|
||||
model_id: Final = string_value(object_value(created["model_info"])["id"])
|
||||
scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (model_id,))
|
||||
return model_id
|
||||
|
||||
|
||||
def _team_model_rows(model_id: str) -> list[dict[str, JsonValue]]:
|
||||
return read_rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (model_id,))
|
||||
|
||||
|
||||
def _delete_while_locked(gateway: Gateway, team_id: str, lock_sql: str, lock_target: str) -> httpx.Response:
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
with psycopg.connect(os.environ["DATABASE_URL"]) as holder:
|
||||
holder.execute(lock_sql, (lock_target,))
|
||||
pending: Final = _post_in_background(pool, gateway, "/team/delete", {"team_ids": [team_id]})
|
||||
eventually(
|
||||
lambda: _oldest_transaction_blocked_by(holder.info.backend_pid),
|
||||
lambda age: age > LOCK_HELD_PAST_DEFAULT_TRANSACTION_SECONDS,
|
||||
seconds=30,
|
||||
)
|
||||
assert not pending.done(), "delete returned while the lock was still held"
|
||||
return pending.result(timeout=60)
|
||||
|
||||
|
||||
def _generate_key(scenario: Scenario, **fields: JsonValue) -> str:
|
||||
"""A key the team delete is expected to remove; cleanup only deletes it if it is still live."""
|
||||
key: Final = string_value(scenario.gateway.post("/key/generate", fields)["key"])
|
||||
|
|
@ -392,6 +438,36 @@ def test_delete_waits_for_the_team_advisory_lock_and_completes_after_release(
|
|||
assert len(_tombstones(team)) == 1, _tombstones(team)
|
||||
|
||||
|
||||
def test_delete_completes_after_waiting_on_a_locked_team_model_row_past_the_default_transaction_timeout(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
team: Final = _create_team(scenario)
|
||||
model_id: Final = _create_team_model(scenario, team)
|
||||
response: Final = _delete_while_locked(gateway, team, LOCK_MODEL_ROW_SQL, model_id)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"deleted_teams": [team]}
|
||||
assert _team_rows(team) == []
|
||||
assert _team_model_rows(model_id) == []
|
||||
assert len(_tombstones(team)) == 1, _tombstones(team)
|
||||
|
||||
|
||||
def test_delete_completes_after_waiting_on_the_team_lock_past_the_default_transaction_timeout(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
user: Final = scenario.user()
|
||||
team: Final = _create_team(scenario)
|
||||
_bulk_member_add(gateway, team, [user])
|
||||
response: Final = _delete_while_locked(gateway, team, TAKE_TEAM_LOCK_SQL, team)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"deleted_teams": [team]}
|
||||
assert _team_rows(team) == []
|
||||
assert _membership_user_ids(team) == []
|
||||
assert _user_teams(user) == []
|
||||
assert len(_tombstones(team)) == 1, _tombstones(team)
|
||||
|
||||
|
||||
def test_deleting_two_teams_sharing_a_member_in_one_call_clears_both_from_the_member(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
user: Final = scenario.user()
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ FirstAttempt = Literal[
|
|||
"drop_after_message_start",
|
||||
"error_frame",
|
||||
"error_frame_after_message_start",
|
||||
"error_frame_after_content",
|
||||
"http_status",
|
||||
"drop_after_content",
|
||||
]
|
||||
|
|
@ -66,6 +67,8 @@ class _Failure:
|
|||
return status_reply(self.status)
|
||||
case "drop_after_content":
|
||||
return dropping_reply(chunks, abort_after=3)
|
||||
case "error_frame_after_content":
|
||||
return stream_reply((*chunks[:3], error_frame(self.status, f"scripted {self.status}")))
|
||||
|
||||
|
||||
_MID_STREAM_FAILURES: Final = (
|
||||
|
|
@ -124,12 +127,12 @@ def _stream(gateway: Gateway, body: Mapping[str, JsonValue]) -> tuple[int, tuple
|
|||
return response.status_code, parse_sse(response.text)
|
||||
|
||||
|
||||
def _rows(request_id: str) -> list[dict[str, JsonValue]]:
|
||||
return read_rows('SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (request_id,))
|
||||
|
||||
|
||||
def _success_rows(request_id: str) -> list[dict[str, JsonValue]]:
|
||||
return eventually(
|
||||
lambda: read_rows('SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (request_id,)),
|
||||
lambda rows: len(rows) >= 1,
|
||||
seconds=70,
|
||||
)
|
||||
return eventually(lambda: _rows(request_id), lambda rows: len(rows) >= 1, seconds=70)
|
||||
|
||||
|
||||
def _assert_completed_by_retry(
|
||||
|
|
@ -141,6 +144,8 @@ def _assert_completed_by_retry(
|
|||
assert delta_text(events) == _TEXT, events
|
||||
assert [request.target for request in wire.drain()] == ["/v1/messages"] * attempts
|
||||
assert _success_rows(_served_id(prompt, attempts)) == [{"status": "success", "model_group": model}]
|
||||
superseded_rows: Final = {attempt: _rows(_served_id(prompt, attempt)) for attempt in range(1, attempts)}
|
||||
assert not any(superseded_rows.values()), superseded_rows
|
||||
|
||||
|
||||
def _assert_rejected_before_content(
|
||||
|
|
@ -302,6 +307,23 @@ def test_stream_dropping_after_content_is_not_retried(gateway: Gateway) -> None:
|
|||
assert [request.target for request in wire.drain()] == ["/v1/messages"]
|
||||
|
||||
|
||||
def test_stream_error_frame_after_content_is_not_retried_and_keeps_its_spend_row(gateway: Gateway) -> None:
|
||||
prompt: Final = _prompt()
|
||||
attempts: Final = Attempts()
|
||||
with (
|
||||
wire_server(_upstream(prompt, _Failure("error_frame_after_content", 529), attempts)) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, num_retries=1)
|
||||
status, events = _stream(gateway, _body(model, prompt, "deployment"))
|
||||
assert status == 200, events
|
||||
assert event_types(events) == (*LIFECYCLE[:3], "error"), events
|
||||
assert message_id(events) == _served_id(prompt, 1), events
|
||||
assert delta_text(events) == _TEXT, events
|
||||
assert [request.target for request in wire.drain()] == ["/v1/messages"]
|
||||
assert [row["model_group"] for row in _success_rows(_served_id(prompt, 1))] == [model]
|
||||
|
||||
|
||||
def test_stream_rejected_with_401_before_it_opens_is_not_retried(gateway: Gateway) -> None:
|
||||
prompt: Final = _prompt()
|
||||
attempts: Final = Attempts()
|
||||
|
|
|
|||
|
|
@ -364,7 +364,7 @@ def test_spend_log_keeps_the_tools_the_caller_sent(rig: Rig, model: str, guardra
|
|||
assert object_value(rows[0]["proxy_server_request"])["tools"] == [strict_tool()]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tools", [7, "x" * 5000, ["lint"]], ids=["int", "five_kb", "list_of_str"])
|
||||
@pytest.mark.parametrize("tools", [7, "x" * 5000, ["lint"], ""], ids=["int", "five_kb", "list_of_str", "empty_string"])
|
||||
def test_a_malformed_tools_field_is_answered_with_an_error_and_the_proxy_keeps_serving(
|
||||
rig: Rig, tools: JsonValue
|
||||
) -> None:
|
||||
|
|
@ -379,14 +379,6 @@ def test_a_malformed_tools_field_is_answered_with_an_error_and_the_proxy_keeps_s
|
|||
assert reply_text(control_status, control_raw, streaming=False) == f"linted {control_nonce.removeprefix('nonce-')}"
|
||||
|
||||
|
||||
def test_an_empty_string_tools_field_reaches_claude_as_sent(rig: Rig) -> None:
|
||||
request_nonce: Final = nonce()
|
||||
status, raw = rig.send(messages_body(CLAUDE_MODEL, request_nonce, tools=""))
|
||||
|
||||
assert reply_text(status, raw, streaming=False) == f"linted {request_nonce.removeprefix('nonce-')}"
|
||||
assert sent_body(rig.received(request_nonce), CLAUDE_TARGET)["tools"] == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "target"),
|
||||
[(CLAUDE_MODEL, CLAUDE_TARGET), (OPENAI_MODEL, RESPONSES_TARGET)],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,308 @@
|
|||
import json
|
||||
import threading
|
||||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.anthropic_sse import message_json
|
||||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.openai_wire import chat_reply
|
||||
from integration._support.ports import allocate_port
|
||||
from integration._support.process import OwnedProxy, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
GEMINI_MODEL: Final = "gemini/gemini-3.8-flash"
|
||||
ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-5-5"
|
||||
OPENAI_MODEL: Final = "openai/gpt-5.6"
|
||||
ECHO_GEMINI_TOOLS: Final = "echo-gemini-tools"
|
||||
NONE_CHAT_ROWS: Final = "none-chat-rows"
|
||||
NONE_CHAT_ROWS_DEFAULT: Final = "none-chat-rows-default"
|
||||
NONE_ANTHROPIC_IMAGE: Final = "none-anthropic-image"
|
||||
ECHO_ANTHROPIC_MESSAGES: Final = "echo-anthropic-messages"
|
||||
FORWARDS_MESSAGES_AS_SENT: Final = frozenset({NONE_CHAT_ROWS, NONE_ANTHROPIC_IMAGE, ECHO_ANTHROPIC_MESSAGES})
|
||||
FAIL_OPEN_CONNECT_ERROR: Final = "fail-open-connect-error"
|
||||
FAIL_OPEN_TIMEOUT: Final = "fail-open-timeout"
|
||||
FAIL_OPEN_HTTP_STATUS: Final = "fail-open-http-status"
|
||||
FAIL_OPEN_LINE: Final = "Generic Guardrail API error (fail-open)"
|
||||
PNG: Final = (
|
||||
"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAFklEQVR4nGO4IydHEmIY1TCqYfhqAACaMxgQdGu1YAAAAABJRU5ErkJggg=="
|
||||
)
|
||||
_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
|
||||
_JSON: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _verdict_none(_request: Request) -> Reply:
|
||||
return Reply(body=json.dumps({"action": "NONE"}).encode())
|
||||
|
||||
|
||||
def _verdict_echo(request: Request) -> Reply:
|
||||
body: Final = _JSON.validate_json(request.body)
|
||||
echoed: Final = {key: body[key] for key in ("structured_messages", "tools") if body.get(key)}
|
||||
return Reply(body=json.dumps({"action": "NONE", **echoed}).encode())
|
||||
|
||||
|
||||
def _unavailable(_request: Request) -> Reply:
|
||||
return Reply(status=503, body=b'{"error": "guardrail unavailable"}')
|
||||
|
||||
|
||||
def _gemini_reply(_request: Request) -> Reply:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"candidates": [{"content": {"role": "model", "parts": [{"text": "ok"}]}, "finishReason": "STOP"}],
|
||||
"usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 1, "totalTokenCount": 4},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_reply(_request: Request) -> Reply:
|
||||
return Reply(body=message_json("msg_guardrail_wire", "claude-sonnet-5-5", "ok"))
|
||||
|
||||
|
||||
def _openai_reply(_request: Request) -> Reply:
|
||||
return chat_reply("chatcmpl-guardrail-wire", "gpt-5.6", "ok", stream=False)
|
||||
|
||||
|
||||
def _guardrail(name: str, api_base: str, **params: JsonValue) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"guardrail_name": name,
|
||||
"litellm_params": {
|
||||
"guardrail": "generic_guardrail_api",
|
||||
"mode": "pre_call",
|
||||
"default_on": False,
|
||||
"api_base": api_base,
|
||||
"api_key": "synthetic-guardrail-key",
|
||||
**params,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GuardrailRig:
|
||||
owned: OwnedProxy
|
||||
policies: Mapping[str, Wire]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[GuardrailRig]:
|
||||
directory: Final = tmp_path_factory.mktemp("generic-guardrail-wire")
|
||||
release: Final = threading.Event()
|
||||
|
||||
def held_until_teardown(request: Request) -> Reply:
|
||||
release.wait()
|
||||
return _verdict_none(request)
|
||||
|
||||
with ExitStack() as stack:
|
||||
policies: Final = MappingProxyType(
|
||||
{
|
||||
ECHO_GEMINI_TOOLS: stack.enter_context(wire_server(_verdict_echo)),
|
||||
NONE_CHAT_ROWS: stack.enter_context(wire_server(_verdict_none)),
|
||||
NONE_CHAT_ROWS_DEFAULT: stack.enter_context(wire_server(_verdict_none)),
|
||||
NONE_ANTHROPIC_IMAGE: stack.enter_context(wire_server(_verdict_none)),
|
||||
ECHO_ANTHROPIC_MESSAGES: stack.enter_context(wire_server(_verdict_echo)),
|
||||
}
|
||||
)
|
||||
stalled: Final = stack.enter_context(wire_server(held_until_teardown))
|
||||
stack.callback(release.set)
|
||||
unavailable: Final = stack.enter_context(wire_server(_unavailable))
|
||||
config: Final = _JSON.validate_python(yaml.safe_load(_CONFIG.read_text()))
|
||||
path: Final = directory / "generic-guardrail-wire.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
**config,
|
||||
"guardrails": [
|
||||
*(
|
||||
_guardrail(name, wire.url, forward_messages_as_sent=name in FORWARDS_MESSAGES_AS_SENT)
|
||||
for name, wire in policies.items()
|
||||
),
|
||||
_guardrail(FAIL_OPEN_CONNECT_ERROR, f"http://127.0.0.1:{allocate_port()}", fail_on_error=False),
|
||||
_guardrail(FAIL_OPEN_TIMEOUT, stalled.url, fail_on_error=False, timeout=1),
|
||||
_guardrail(FAIL_OPEN_HTTP_STATUS, unavailable.url, fail_on_error=False),
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
owned: Final = stack.enter_context(owned_proxy_process(session_gateway, directory, {}, config=path))
|
||||
yield GuardrailRig(owned, policies)
|
||||
|
||||
|
||||
def _bodies(wire: Wire) -> list[dict[str, JsonValue]]:
|
||||
return [_JSON.validate_json(request.body) for request in wire.drain()]
|
||||
|
||||
|
||||
def _fail_open_lines(log: Path, guardrail: str) -> list[str]:
|
||||
return [
|
||||
line
|
||||
for line in log.read_text(errors="replace").splitlines()
|
||||
if FAIL_OPEN_LINE in line and f"guardrail_name={guardrail} " in line
|
||||
]
|
||||
|
||||
|
||||
def test_gemini_native_tools_reach_the_guardrail_and_gemini_as_sent(rig: GuardrailRig) -> None:
|
||||
tools: Final = [
|
||||
{"function_declarations": [{"name": "get_weather", "parameters": {"type": "object", "properties": {}}}]},
|
||||
{"googleSearch": {}},
|
||||
]
|
||||
with wire_server(_gemini_reply) as gemini, rig.owned.gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=GEMINI_MODEL, api_base=gemini.url, api_key="synthetic-gemini-key")
|
||||
response: Final = rig.owned.gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "weather in Paris?"}],
|
||||
"tools": tools,
|
||||
"guardrails": [ECHO_GEMINI_TOOLS],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert [body["tools"] for body in _bodies(rig.policies[ECHO_GEMINI_TOOLS])] == [tools]
|
||||
sent: Final = _bodies(gemini)
|
||||
assert len(sent) == 1, sent
|
||||
assert {"googleSearch": {}} in sent[0]["tools"], sent
|
||||
assert [tool for tool in sent[0]["tools"] if "type" in tool] == [], sent
|
||||
|
||||
|
||||
CHAT_ROWS: Final = (
|
||||
{"role": "user", "name": "pat", "content": "weather in Paris?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"index": 0, "id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
|
||||
)
|
||||
OPENAI_CHAT_ROWS: Final = (
|
||||
{"role": "user", "content": "weather in Paris?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}}],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("guardrail", "seen"),
|
||||
[(NONE_CHAT_ROWS, CHAT_ROWS), (NONE_CHAT_ROWS_DEFAULT, OPENAI_CHAT_ROWS)],
|
||||
ids=["forward_messages_as_sent", "default"],
|
||||
)
|
||||
def test_guardrail_sees_chat_rows_as_sent_only_when_configured(
|
||||
rig: GuardrailRig, guardrail: str, seen: tuple[dict[str, JsonValue], ...]
|
||||
) -> None:
|
||||
messages: Final = list(CHAT_ROWS)
|
||||
with wire_server(_openai_reply) as openai_peer, rig.owned.gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=OPENAI_MODEL, api_base=f"{openai_peer.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
response: Final = rig.owned.gateway.request(
|
||||
"POST", "/v1/chat/completions", {"model": model, "messages": messages, "guardrails": [guardrail]}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert [body["structured_messages"] for body in _bodies(rig.policies[guardrail])] == [list(seen)]
|
||||
assert len(openai_peer.drain()) == 1
|
||||
|
||||
|
||||
def test_guardrail_sees_an_anthropic_image_block_on_chat_completions_as_sent(rig: GuardrailRig) -> None:
|
||||
image: Final = [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG}}]
|
||||
with wire_server(_anthropic_reply) as anthropic_peer, rig.owned.gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=ANTHROPIC_MODEL, api_base=anthropic_peer.url, api_key="synthetic-anthropic-key"
|
||||
)
|
||||
rig.owned.gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": image}, {"role": "user", "content": "what color?"}],
|
||||
"guardrails": [NONE_ANTHROPIC_IMAGE],
|
||||
},
|
||||
)
|
||||
seen: Final = _bodies(rig.policies[NONE_ANTHROPIC_IMAGE])
|
||||
assert [body["structured_messages"][0]["content"] for body in seen] == [image], seen
|
||||
|
||||
|
||||
def test_echoing_guardrail_leaves_the_anthropic_messages_request_unchanged(rig: GuardrailRig) -> None:
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What color is this square? Also check the weather in Paris."},
|
||||
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG}},
|
||||
{"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Trip notes."}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}}],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_1", "content": "sunny, 21C"},
|
||||
{"type": "text", "text": "Answer both in one sentence."},
|
||||
],
|
||||
},
|
||||
]
|
||||
tools: Final = [
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Weather for a city",
|
||||
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
|
||||
}
|
||||
]
|
||||
with wire_server(_anthropic_reply) as anthropic_peer, rig.owned.gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=ANTHROPIC_MODEL, api_base=anthropic_peer.url, api_key="synthetic-anthropic-key"
|
||||
)
|
||||
response: Final = rig.owned.gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 64,
|
||||
"messages": messages,
|
||||
"tools": tools,
|
||||
"guardrails": [ECHO_ANTHROPIC_MESSAGES],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(rig.policies[ECHO_ANTHROPIC_MESSAGES].drain()) == 1
|
||||
assert [body["messages"] for body in _bodies(anthropic_peer)] == [messages]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("guardrail", "tokens"),
|
||||
[
|
||||
(FAIL_OPEN_CONNECT_ERROR, ("error_type=ConnectError",)),
|
||||
(FAIL_OPEN_TIMEOUT, ("error_type=Timeout",)),
|
||||
(FAIL_OPEN_HTTP_STATUS, ("error_type=HTTPStatusError", "http_status_code=503")),
|
||||
],
|
||||
ids=["connect_error", "timeout", "http_status"],
|
||||
)
|
||||
def test_fail_open_log_line_names_the_error_type(rig: GuardrailRig, guardrail: str, tokens: tuple[str, ...]) -> None:
|
||||
with wire_server(_openai_reply) as openai_peer, rig.owned.gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=OPENAI_MODEL, api_base=f"{openai_peer.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
response: Final = rig.owned.gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": "hi"}], "guardrails": [guardrail]},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(openai_peer.drain()) == 1
|
||||
lines: Final = eventually(lambda: _fail_open_lines(rig.owned.log, guardrail), bool, seconds=30)
|
||||
assert [line for line in lines if not all(token in line for token in tokens)] == [], lines
|
||||
|
|
@ -1147,8 +1147,8 @@ def test_messages_count_tokens_surfaces_the_mantle_error_for_a_non_text_system(
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
"tools",
|
||||
["", "x" * 5120, ["get_weather"]],
|
||||
ids=["empty_string", "5kb_string", "list_of_strings"],
|
||||
[5, "", "x" * 5120, ["get_weather"]],
|
||||
ids=["int", "empty_string", "5kb_string", "list_of_strings"],
|
||||
)
|
||||
def test_messages_count_tokens_rejects_malformed_tools_without_calling_either_peer(
|
||||
counting_proxy: OwnedProxy, mantle_port: int, tools: JsonValue
|
||||
|
|
@ -1161,8 +1161,8 @@ def test_messages_count_tokens_rejects_malformed_tools_without_calling_either_pe
|
|||
):
|
||||
model: Final = _deployment(scenario, runtime.url)
|
||||
refused: Final = _count(gateway, {**_bare(model), "tools": tools})
|
||||
assert 400 <= refused.status_code < 600, refused.text
|
||||
assert "input_tokens" not in refused.text, refused.text
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert object_value(_JSON_OBJECT.validate_json(refused.content)["error"])["type"] == "invalid_request_error"
|
||||
assert runtime.drain() == () and mantle.drain() == ()
|
||||
assert _generated_then_counted(gateway.client, gateway.key, model, _bare(model)) == (200, 200, _MANTLE_COUNT)
|
||||
assert _mantle_bodies(mantle) == (_MANTLE_BARE,)
|
||||
|
|
@ -1185,8 +1185,13 @@ def test_messages_count_tokens_without_messages_is_rejected(
|
|||
assert runtime.drain() == () and mantle.drain() == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"fields",
|
||||
[{}, {"tools": 5}, {"web_search_options": None}],
|
||||
ids=["well_formed", "int_tools", "null_web_search_options"],
|
||||
)
|
||||
def test_messages_count_tokens_unauthenticated_request_never_reaches_either_peer(
|
||||
counting_proxy: OwnedProxy, mantle_port: int
|
||||
counting_proxy: OwnedProxy, mantle_port: int, fields: dict[str, JsonValue]
|
||||
) -> None:
|
||||
gateway: Final = counting_proxy.gateway
|
||||
with (
|
||||
|
|
@ -1196,7 +1201,7 @@ def test_messages_count_tokens_unauthenticated_request_never_reaches_either_peer
|
|||
):
|
||||
model: Final = _deployment(scenario, runtime.url)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/messages/count_tokens", _full(model), key="sk-not-a-key-this-proxy-issued"
|
||||
"POST", "/v1/messages/count_tokens", {**_full(model), **fields}, key="sk-not-a-key-this-proxy-issued"
|
||||
)
|
||||
assert response.status_code == 401, response.text
|
||||
assert "input_tokens" not in response.text, response.text
|
||||
|
|
|
|||
619
tests/integration/providers/test_malformed_tools_chaos.py
Normal file
619
tests/integration/providers/test_malformed_tools_chaos.py
Normal file
|
|
@ -0,0 +1,619 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from queue import SimpleQueue
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
import httpx
|
||||
import psutil
|
||||
import pytest
|
||||
from integration._support.anthropic_sse import error_body, message_json, message_stream
|
||||
from integration._support.client import Gateway, eventually, gateway_from_environment
|
||||
from integration._support.database import write_rows
|
||||
from integration._support.openai_wire import chat_reply, responses_reply
|
||||
from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from integration._support.worker_readiness import wait_for_worker_readiness
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_OPENAI_BACKEND: Final = "gpt-4o-mini"
|
||||
_CLAUDE_BACKEND: Final = "claude-haiku-4-5"
|
||||
_GEMINI_BACKEND: Final = "gemini-2.5-flash"
|
||||
_PROVIDER_KEY: Final = "scripted-malformed-tools-chaos-key"
|
||||
_OWNED_CHAT: Final = "malformed-tools-owned-chat"
|
||||
_OWNED_STORE: Final = "vs_malformed_tools_owned_store"
|
||||
_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})")
|
||||
_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
|
||||
_OWNED_PROXY_CELL_SECONDS: Final = 2 * graceful_stop_seconds() + 120
|
||||
_RESPAWN_SECONDS: Final = graceful_stop_seconds() + 60
|
||||
_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1")))
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_NOT_A_LIST: Final = "tools: Input should be a valid list"
|
||||
_WEATHER_SCHEMA: Final[dict[str, JsonValue]] = {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
}
|
||||
_CHAT_TOOL: Final[dict[str, JsonValue]] = {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "description": "Look up the weather", "parameters": _WEATHER_SCHEMA},
|
||||
}
|
||||
_CLAUDE_TOOL: Final[dict[str, JsonValue]] = {
|
||||
"name": "get_weather",
|
||||
"description": "Look up the weather",
|
||||
"input_schema": _WEATHER_SCHEMA,
|
||||
}
|
||||
_RESPONSES_TOOL: Final[dict[str, JsonValue]] = {
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"description": "Look up the weather",
|
||||
"parameters": _WEATHER_SCHEMA,
|
||||
}
|
||||
_GEMINI_TOOL: Final[dict[str, JsonValue]] = {
|
||||
"functionDeclarations": [
|
||||
{"name": "get_weather", "description": "Look up the weather", "parameters": _WEATHER_SCHEMA}
|
||||
]
|
||||
}
|
||||
Kind: TypeAlias = Literal["valid", "rejected", "dropped"]
|
||||
|
||||
|
||||
def _marker(label: str) -> str:
|
||||
return f"{label} marker-{uuid.uuid4().hex}"
|
||||
|
||||
|
||||
def _tag(text: str) -> str:
|
||||
found: Final = _MARKER.search(text)
|
||||
return found.group(1) if found else "none"
|
||||
|
||||
|
||||
def _answer(tag: str) -> str:
|
||||
return f"answer {tag}"
|
||||
|
||||
|
||||
def _gemini_body(tag: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"candidates": [
|
||||
{"content": {"role": "model", "parts": [{"text": _answer(tag)}]}, "index": 0, "finishReason": "STOP"}
|
||||
],
|
||||
"usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 2, "totalTokenCount": 11},
|
||||
"modelVersion": _GEMINI_BACKEND,
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _scripted(request: Request) -> Reply:
|
||||
text: Final = request.body.decode()
|
||||
tag: Final = _tag(text)
|
||||
path: Final = request.target.split("?")[0]
|
||||
stream: Final = bool(text) and _JSON_OBJECT.validate_json(request.body).get("stream") is True
|
||||
if path == "/v1/chat/completions":
|
||||
return chat_reply(f"chatcmpl-{tag}", _OPENAI_BACKEND, _answer(tag), stream=stream)
|
||||
if path == "/v1/responses":
|
||||
return responses_reply(f"resp_{tag}", _OPENAI_BACKEND, _answer(tag), stream=stream)
|
||||
if path == "/v1/messages" and _JSON_OBJECT.validate_json(request.body).get("tools") == 5:
|
||||
return Reply(status=400, body=error_body(400, _NOT_A_LIST))
|
||||
if path == "/v1/messages" and stream:
|
||||
return Reply(
|
||||
content_type="text/event-stream", chunks=message_stream(f"msg_{tag}", _CLAUDE_BACKEND, _answer(tag))
|
||||
)
|
||||
if path == "/v1/messages":
|
||||
return Reply(body=message_json(f"msg_{tag}", _CLAUDE_BACKEND, _answer(tag)))
|
||||
if path.endswith(":streamGenerateContent"):
|
||||
return Reply(content_type="text/event-stream", chunks=(b"data: " + _gemini_body(tag) + b"\r\n\r\n",))
|
||||
if path.endswith(":generateContent"):
|
||||
return Reply(body=_gemini_body(tag))
|
||||
return Reply(status=404, body=json.dumps({"error": f"unscripted {request.target}"}).encode())
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Switchboard:
|
||||
down: threading.Event
|
||||
holding: threading.Event
|
||||
release: threading.Event
|
||||
held: SimpleQueue[str]
|
||||
|
||||
def respond(self, request: Request) -> Reply:
|
||||
if self.down.is_set():
|
||||
return Reply(drop_connection=True)
|
||||
if self.holding.is_set():
|
||||
self.held.put(request.target)
|
||||
assert self.release.wait(timeout=120), "Held request was never released"
|
||||
return _scripted(request)
|
||||
|
||||
def reset(self) -> None:
|
||||
self.down.clear()
|
||||
self.release.set()
|
||||
self.holding.clear()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Rig:
|
||||
gateway: Gateway
|
||||
wire: Wire
|
||||
switchboard: _Switchboard
|
||||
chat: str
|
||||
claude: str
|
||||
gemini: str
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig() -> Iterator[_Rig]:
|
||||
switchboard: Final = _Switchboard(threading.Event(), threading.Event(), threading.Event(), SimpleQueue())
|
||||
with (
|
||||
gateway_from_environment() as gateway,
|
||||
wire_server(switchboard.respond) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
built: Final = _Rig(
|
||||
gateway=gateway,
|
||||
wire=wire,
|
||||
switchboard=switchboard,
|
||||
chat=scenario.model(api_base=f"{wire.url}/v1", api_key=_PROVIDER_KEY),
|
||||
claude=scenario.model(model=f"anthropic/{_CLAUDE_BACKEND}", api_base=wire.url, api_key=_PROVIDER_KEY),
|
||||
gemini=scenario.model(model=f"gemini/{_GEMINI_BACKEND}", api_base=wire.url, api_key=_PROVIDER_KEY),
|
||||
)
|
||||
for group in (built.chat, built.claude, built.gemini):
|
||||
wait_for_worker_readiness(gateway, group, 1, _PROXY_WORKERS)
|
||||
try:
|
||||
yield built
|
||||
finally:
|
||||
switchboard.reset()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset(rig: _Rig) -> Iterator[None]:
|
||||
rig.switchboard.reset()
|
||||
rig.wire.drain()
|
||||
yield
|
||||
rig.switchboard.reset()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Call:
|
||||
path: str
|
||||
body: dict[str, JsonValue]
|
||||
marker: str
|
||||
kind: Kind
|
||||
|
||||
|
||||
def _tools_for(kind: Kind, tool: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
if kind == "rejected":
|
||||
return {"tools": 5}
|
||||
if kind == "dropped":
|
||||
return {"tools": [tool, 5], "drop_params": True}
|
||||
return {"tools": [tool]}
|
||||
|
||||
|
||||
def _call(rig: _Rig, index: int, kind: Kind, label: str) -> _Call:
|
||||
marker: Final = _marker(label)
|
||||
stream: Final = (index // 4) % 2 == 1
|
||||
family: Final = index % 4
|
||||
if family == 0:
|
||||
return _Call(
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": rig.chat,
|
||||
"messages": [{"role": "user", "content": marker}],
|
||||
"stream": stream,
|
||||
**_tools_for(kind, _CHAT_TOOL),
|
||||
},
|
||||
marker,
|
||||
kind,
|
||||
)
|
||||
if family == 1:
|
||||
return _Call(
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": rig.claude,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": marker}],
|
||||
"stream": stream,
|
||||
**_tools_for(kind, _CLAUDE_TOOL),
|
||||
},
|
||||
marker,
|
||||
kind,
|
||||
)
|
||||
if family == 2:
|
||||
return _Call(
|
||||
"/v1/responses",
|
||||
{"model": rig.chat, "input": marker, "stream": stream, **_tools_for(kind, _RESPONSES_TOOL)},
|
||||
marker,
|
||||
kind,
|
||||
)
|
||||
action: Final = "streamGenerateContent?alt=sse" if stream else "generateContent"
|
||||
return _Call(
|
||||
f"/v1beta/models/{rig.gemini}:{action}",
|
||||
{"contents": [{"role": "user", "parts": [{"text": marker}]}], **_tools_for(kind, _GEMINI_TOOL)},
|
||||
marker,
|
||||
kind,
|
||||
)
|
||||
|
||||
|
||||
def _kind(index: int) -> Kind:
|
||||
kinds: Final[tuple[Kind, ...]] = ("valid", "rejected", "dropped")
|
||||
return kinds[index % 3]
|
||||
|
||||
|
||||
def _clients(stack: ExitStack, base_url: str, count: int) -> tuple[httpx.Client, ...]:
|
||||
return tuple(
|
||||
stack.enter_context(httpx.Client(base_url=base_url, timeout=120, trust_env=False)) for _ in range(count)
|
||||
)
|
||||
|
||||
|
||||
def _post(client: httpx.Client, key: str, call: _Call) -> httpx.Response:
|
||||
return client.post(call.path, json=call.body, headers={"Authorization": f"Bearer {key}"})
|
||||
|
||||
|
||||
def _status_or_dropped(client: httpx.Client, key: str, call: _Call) -> int | None:
|
||||
try:
|
||||
return _post(client, key, call).status_code
|
||||
except httpx.TransportError:
|
||||
return None
|
||||
|
||||
|
||||
def _deliveries(wire: Wire) -> tuple[str, ...]:
|
||||
return tuple(request.body.decode() for request in wire.drain())
|
||||
|
||||
|
||||
def _seen(deliveries: tuple[str, ...], marker: str) -> int:
|
||||
return sum(marker in body for body in deliveries)
|
||||
|
||||
|
||||
def _delivered_tool_names(deliveries: tuple[str, ...], marker: str) -> tuple[str, ...]:
|
||||
(body,) = tuple(body for body in deliveries if marker in body)
|
||||
return tuple(re.findall(r'"name": ?"([a-z_]+)"', body))
|
||||
|
||||
|
||||
def _assert_wave(calls: tuple[_Call, ...], statuses: tuple[int | None, ...], deliveries: tuple[str, ...]) -> None:
|
||||
expected_statuses: Final = tuple(400 if call.kind == "rejected" else 200 for call in calls)
|
||||
assert statuses == expected_statuses, tuple(zip(statuses, (call.marker for call in calls), strict=True))
|
||||
seen: Final = tuple(_seen(deliveries, call.marker) for call in calls)
|
||||
assert seen == tuple(0 if call.kind == "rejected" else 1 for call in calls), seen
|
||||
names: Final = tuple(_delivered_tool_names(deliveries, call.marker) for call in calls if call.kind != "rejected")
|
||||
assert all("get_weather" in found for found in names), names
|
||||
|
||||
|
||||
def test_mixed_burst_rejects_malformed_tools_and_forwards_the_rest_once(rig: _Rig) -> None:
|
||||
with ExitStack() as stack:
|
||||
calls: Final = tuple(_call(rig, index, _kind(index), "burst") for index in range(40))
|
||||
clients: Final = _clients(stack, str(rig.gateway.client.base_url), len(calls))
|
||||
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(calls)))
|
||||
statuses: Final = tuple(
|
||||
pool.map(
|
||||
lambda pair: _status_or_dropped(pair[0], rig.gateway.key, pair[1]), zip(clients, calls, strict=True)
|
||||
)
|
||||
)
|
||||
_assert_wave(calls, statuses, _deliveries(rig.wire))
|
||||
|
||||
|
||||
def test_upstream_outage_still_rejects_malformed_tools_and_recovers(rig: _Rig) -> None:
|
||||
with ExitStack() as stack:
|
||||
clients: Final = _clients(stack, str(rig.gateway.client.base_url), 12)
|
||||
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients)))
|
||||
|
||||
def wave(label: str) -> tuple[tuple[_Call, ...], tuple[int | None, ...]]:
|
||||
calls: Final = tuple(_call(rig, index, _kind(index), label) for index in range(len(clients)))
|
||||
statuses: Final = tuple(
|
||||
pool.map(
|
||||
lambda pair: _status_or_dropped(pair[0], rig.gateway.key, pair[1]), zip(clients, calls, strict=True)
|
||||
)
|
||||
)
|
||||
return calls, statuses
|
||||
|
||||
before_calls, before_statuses = wave("before outage")
|
||||
_assert_wave(before_calls, before_statuses, _deliveries(rig.wire))
|
||||
rig.switchboard.down.set()
|
||||
outage_calls, outage_statuses = wave("during outage")
|
||||
for call, status in zip(outage_calls, outage_statuses, strict=True):
|
||||
assert (status == 400) if call.kind == "rejected" else (status is not None and status >= 500), (
|
||||
call.kind,
|
||||
status,
|
||||
)
|
||||
assert rig.gateway.request("GET", "/health/liveliness").status_code == 200
|
||||
rig.wire.drain()
|
||||
rig.switchboard.down.clear()
|
||||
after_calls, after_statuses = wave("after outage")
|
||||
_assert_wave(after_calls, after_statuses, _deliveries(rig.wire))
|
||||
|
||||
|
||||
def test_held_upstream_does_not_delay_malformed_tools_rejections(rig: _Rig) -> None:
|
||||
with ExitStack() as stack:
|
||||
held_calls: Final = tuple(_call(rig, index * 4, "valid", "held") for index in range(8))
|
||||
rejected_calls: Final = tuple(_call(rig, index, "rejected", "rejected") for index in range(8))
|
||||
clients: Final = _clients(stack, str(rig.gateway.client.base_url), len(held_calls) + len(rejected_calls))
|
||||
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients)))
|
||||
rig.switchboard.release.clear()
|
||||
rig.switchboard.holding.set()
|
||||
stack.callback(rig.switchboard.release.set)
|
||||
pending: Final[tuple[Future[int | None], ...]] = tuple(
|
||||
pool.submit(_status_or_dropped, client, rig.gateway.key, call)
|
||||
for client, call in zip(clients[: len(held_calls)], held_calls, strict=True)
|
||||
)
|
||||
eventually(rig.switchboard.held.qsize, lambda size: size == len(held_calls), seconds=60)
|
||||
rejected: Final = tuple(
|
||||
pool.map(
|
||||
lambda pair: _status_or_dropped(pair[0], rig.gateway.key, pair[1]),
|
||||
zip(clients[len(held_calls) :], rejected_calls, strict=True),
|
||||
)
|
||||
)
|
||||
assert rejected == (400,) * len(rejected_calls), rejected
|
||||
assert not any(future.done() for future in pending), "a held request finished before its release"
|
||||
rig.switchboard.release.set()
|
||||
assert tuple(future.result(timeout=120) for future in pending) == (200,) * len(held_calls)
|
||||
deliveries: Final = _deliveries(rig.wire)
|
||||
assert tuple(_seen(deliveries, call.marker) for call in held_calls) == (1,) * len(held_calls)
|
||||
assert tuple(_seen(deliveries, call.marker) for call in rejected_calls) == (0,) * len(rejected_calls)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Owned:
|
||||
proxy: OwnedProxy
|
||||
wire: Wire
|
||||
held: SimpleQueue[str]
|
||||
holding: threading.Event
|
||||
release: threading.Event
|
||||
blocked_tool: str
|
||||
denied_tool: str
|
||||
|
||||
|
||||
def _owned_config(path: Path, upstream: str, denied_tool: str) -> Path:
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": _OWNED_CHAT,
|
||||
"litellm_params": {
|
||||
"model": f"openai/{_OPENAI_BACKEND}",
|
||||
"api_base": f"{upstream}/v1",
|
||||
"api_key": _PROVIDER_KEY,
|
||||
},
|
||||
}
|
||||
],
|
||||
"general_settings": {
|
||||
"master_key": "os.environ/LITELLM_MASTER_KEY",
|
||||
"database_url": "os.environ/DATABASE_URL",
|
||||
},
|
||||
"litellm_settings": {"drop_params": True},
|
||||
"router_settings": {"disable_cooldowns": True},
|
||||
"guardrails": [
|
||||
{
|
||||
"guardrail_name": "malformed-tools-policy",
|
||||
"litellm_params": {"guardrail": "tool_policy", "mode": "pre_call", "default_on": True},
|
||||
},
|
||||
{
|
||||
"guardrail_name": "malformed-tools-permission",
|
||||
"litellm_params": {
|
||||
"guardrail": "tool_permission",
|
||||
"mode": "pre_call",
|
||||
"default_on": False,
|
||||
"default_action": "allow",
|
||||
"on_disallowed_action": "block",
|
||||
"rules": [{"id": "deny_tool", "tool_name": f"^{denied_tool}$", "decision": "deny"}],
|
||||
},
|
||||
},
|
||||
],
|
||||
"vector_store_registry": [
|
||||
{
|
||||
"vector_store_name": "malformed-tools-owned-store",
|
||||
"litellm_params": {
|
||||
"vector_store_id": _OWNED_STORE,
|
||||
"custom_llm_provider": "openai",
|
||||
"api_base": upstream,
|
||||
"api_key": _PROVIDER_KEY,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def owned(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Owned]:
|
||||
suffix: Final = uuid.uuid4().hex[:12]
|
||||
blocked_tool: Final = f"wipe_disk_{suffix}"
|
||||
denied_tool: Final = f"run_shell_{suffix}"
|
||||
held: Final[SimpleQueue[str]] = SimpleQueue()
|
||||
holding: Final = threading.Event()
|
||||
release: Final = threading.Event()
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
if holding.is_set():
|
||||
held.put(request.target)
|
||||
assert release.wait(timeout=120), "Held request was never released"
|
||||
return _scripted(request)
|
||||
|
||||
directory: Final = tmp_path_factory.mktemp("malformed-tools-owned")
|
||||
with ExitStack() as stack:
|
||||
gateway: Final = stack.enter_context(gateway_from_environment())
|
||||
policy: Final = gateway.request(
|
||||
"POST", "/v1/tool/policy", {"tool_name": blocked_tool, "input_policy": "blocked"}
|
||||
)
|
||||
assert policy.status_code == 200, policy.text
|
||||
stack.callback(write_rows, 'DELETE FROM "LiteLLM_ToolTable" WHERE tool_name = %s', (blocked_tool,))
|
||||
wire: Final = stack.enter_context(wire_server(respond))
|
||||
stack.callback(release.set)
|
||||
proxy: Final = stack.enter_context(
|
||||
owned_proxy_process(
|
||||
gateway,
|
||||
directory,
|
||||
{"ANTHROPIC_API_BASE": wire.url, "ANTHROPIC_API_KEY": _PROVIDER_KEY},
|
||||
config=_owned_config(directory / "malformed-tools.yaml", wire.url, denied_tool),
|
||||
workers=2,
|
||||
)
|
||||
)
|
||||
yield _Owned(proxy, wire, held, holding, release, blocked_tool, denied_tool)
|
||||
|
||||
|
||||
def _owned_chat(owned: _Owned, marker: str, **fields: JsonValue) -> httpx.Response:
|
||||
return owned.proxy.gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": _OWNED_CHAT, "messages": [{"role": "user", "content": marker}], **fields},
|
||||
)
|
||||
|
||||
|
||||
def _named_tool(name: str) -> dict[str, JsonValue]:
|
||||
return {"type": "function", "function": {"name": name, "parameters": _WEATHER_SCHEMA}}
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
|
||||
def test_global_drop_params_wins_over_an_explicit_request_false(owned: _Owned) -> None:
|
||||
marker: Final = _marker("global drop")
|
||||
response: Final = _owned_chat(owned, marker, tools=5, drop_params=False)
|
||||
assert response.status_code == 200, response.text
|
||||
assert _answer(_tag(marker)) in response.text, response.text
|
||||
(sent,) = tuple(request for request in owned.wire.drain() if marker in request.body.decode())
|
||||
assert "tools" not in _JSON_OBJECT.validate_json(sent.body), sent.body
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
|
||||
def test_tool_policy_still_blocks_a_tool_kept_beside_a_dropped_entry(owned: _Owned) -> None:
|
||||
marker: Final = _marker("tool policy")
|
||||
response: Final = _owned_chat(owned, marker, tools=[_named_tool(owned.blocked_tool), 5])
|
||||
assert response.status_code == 400, response.text
|
||||
assert "Violated tool policy" in response.text and owned.blocked_tool in response.text, response.text
|
||||
assert tuple(request for request in owned.wire.drain() if marker in request.body.decode()) == ()
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
|
||||
def test_tool_permission_still_denies_a_tool_kept_beside_a_dropped_entry(owned: _Owned) -> None:
|
||||
marker: Final = _marker("tool permission")
|
||||
response: Final = _owned_chat(
|
||||
owned, marker, tools=[_named_tool(owned.denied_tool), "x"], guardrails=["malformed-tools-permission"]
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
assert "Violated guardrail policy" in response.text and owned.denied_tool in response.text, response.text
|
||||
assert tuple(request for request in owned.wire.drain() if marker in request.body.decode()) == ()
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
|
||||
def test_anthropic_pass_through_forwards_malformed_tools_unchanged(owned: _Owned) -> None:
|
||||
marker: Final = _marker("pass-through")
|
||||
body: Final[dict[str, JsonValue]] = {
|
||||
"model": _CLAUDE_BACKEND,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": marker}],
|
||||
"tools": 5,
|
||||
}
|
||||
response: Final = owned.proxy.gateway.request("POST", "/anthropic/v1/messages", body)
|
||||
assert response.status_code == 400, response.text
|
||||
assert _JSON_OBJECT.validate_json(response.content) == _JSON_OBJECT.validate_json(error_body(400, _NOT_A_LIST))
|
||||
(sent,) = tuple(request for request in owned.wire.drain() if marker in request.body.decode())
|
||||
assert sent.target == "/v1/messages", sent.target
|
||||
assert _JSON_OBJECT.validate_json(sent.body) == body, sent.body
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
|
||||
def test_anthropic_pass_through_streams_well_formed_tools(owned: _Owned) -> None:
|
||||
marker: Final = _marker("pass-through stream")
|
||||
body: Final[dict[str, JsonValue]] = {
|
||||
"model": _CLAUDE_BACKEND,
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": marker}],
|
||||
"tools": [_CLAUDE_TOOL],
|
||||
"stream": True,
|
||||
}
|
||||
response: Final = owned.proxy.gateway.request("POST", "/anthropic/v1/messages", body)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.content == b"".join(message_stream(f"msg_{_tag(marker)}", _CLAUDE_BACKEND, _answer(_tag(marker))))
|
||||
(sent,) = tuple(request for request in owned.wire.drain() if marker in request.body.decode())
|
||||
assert _JSON_OBJECT.validate_json(sent.body) == body, sent.body
|
||||
|
||||
|
||||
def _local_port(client: httpx.Client) -> int:
|
||||
with client.stream("GET", "/health/liveliness") as response:
|
||||
port: Final = int(response.extensions["network_stream"].get_extra_info("client_addr")[1])
|
||||
response.read()
|
||||
assert response.status_code == 200, response.text
|
||||
return port
|
||||
|
||||
|
||||
def _accepted_client_ports(pid: int, proxy_port: int) -> frozenset[int]:
|
||||
return frozenset(
|
||||
connection.raddr.port
|
||||
for connection in psutil.Process(pid).net_connections(kind="tcp")
|
||||
if connection.raddr and connection.laddr.port == proxy_port
|
||||
)
|
||||
|
||||
|
||||
def _owned_call(index: int, label: str) -> _Call:
|
||||
marker: Final = _marker(label)
|
||||
tools: Final[JsonValue] = [_CHAT_TOOL, 5] if index % 2 else [_CHAT_TOOL]
|
||||
return _Call(
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": _OWNED_CHAT,
|
||||
"messages": [{"role": "user", "content": marker}],
|
||||
"stream": index % 4 == 1,
|
||||
"tools": tools,
|
||||
},
|
||||
marker,
|
||||
"dropped" if index % 2 else "valid",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
|
||||
def test_worker_sigkill_mid_burst_leaves_the_replacement_dropping_bad_entries(owned: _Owned) -> None:
|
||||
proxy_url: Final = owned.proxy.gateway.client.base_url
|
||||
workers: Final = eventually(
|
||||
lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.proxy.log.read_text())),
|
||||
lambda pids: len(pids) >= 2,
|
||||
seconds=30,
|
||||
)
|
||||
started_before: Final = len(workers)
|
||||
serving: Final = owned.proxy.worker_pids()
|
||||
owned.wire.drain()
|
||||
with ExitStack() as stack:
|
||||
clients: Final = _clients(stack, str(proxy_url), 12)
|
||||
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients)))
|
||||
ports: Final = tuple(_local_port(client) for client in clients)
|
||||
calls: Final = tuple(_owned_call(index, "worker kill") for index in range(len(clients)))
|
||||
owned.release.clear()
|
||||
owned.holding.set()
|
||||
stack.callback(owned.holding.clear)
|
||||
stack.callback(owned.release.set)
|
||||
futures: Final = tuple(
|
||||
pool.submit(_status_or_dropped, client, owned.proxy.gateway.key, call)
|
||||
for client, call in zip(clients, calls, strict=True)
|
||||
)
|
||||
eventually(owned.held.qsize, lambda size: size == len(clients), seconds=60)
|
||||
shares: Final = {pid: _accepted_client_ports(pid, proxy_url.port or 0) & frozenset(ports) for pid in serving}
|
||||
assert sum(map(len, shares.values())) == len(clients), shares
|
||||
victim: Final = min((pid for pid in serving if shares[pid]), key=lambda pid: len(shares[pid]))
|
||||
with owned.proxy.replacing_worker(victim) as kill:
|
||||
kill()
|
||||
owned.holding.clear()
|
||||
owned.release.set()
|
||||
results: Final = tuple(future.result(timeout=120) for future in futures)
|
||||
for port, result in zip(ports, results, strict=True):
|
||||
assert result == (None if port in shares[victim] else 200), (port, result, shares)
|
||||
eventually(
|
||||
lambda: len(_STARTED_WORKER.findall(owned.proxy.log.read_text())),
|
||||
lambda started: started > started_before,
|
||||
_RESPAWN_SECONDS,
|
||||
)
|
||||
second_wave: Final = tuple(_owned_call(index * 2 + 1, "after kill") for index in range(6))
|
||||
(fresh,) = _clients(stack, str(proxy_url), 1)
|
||||
assert tuple(_post(fresh, owned.proxy.gateway.key, call).status_code for call in second_wave) == (200,) * len(
|
||||
second_wave
|
||||
)
|
||||
deliveries: Final = _deliveries(owned.wire)
|
||||
everything: Final = calls + second_wave
|
||||
assert tuple(_seen(deliveries, call.marker) for call in everything) == (1,) * len(everything)
|
||||
assert tuple(_delivered_tool_names(deliveries, call.marker) for call in everything) == (
|
||||
("get_weather",),
|
||||
) * len(everything)
|
||||
assert owned.proxy.process.poll() is None
|
||||
2731
tests/integration/providers/test_malformed_tools_wire.py
Normal file
2731
tests/integration/providers/test_malformed_tools_wire.py
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1,21 +1,19 @@
|
|||
import json
|
||||
import re
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import ExitStack
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import psutil
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, eventually, held, object_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy_process
|
||||
from integration._support.process import WorkerHealthcheck, graceful_stop_seconds, owned_proxy_process
|
||||
from integration._support.vertex import service_account_json
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
|
@ -38,7 +36,6 @@ _GEMINI_USAGE: Final = {
|
|||
"totalTokenCount": _PROMPT_TOKENS + _COMPLETION_TOKENS,
|
||||
}
|
||||
_ZERO_USAGE: Final = {"promptTokenCount": 0, "candidatesTokenCount": 0, "totalTokenCount": 0}
|
||||
_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
|
||||
_BURST_SELECT: Final = (
|
||||
'SELECT api_key, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs"'
|
||||
" WHERE api_key = ANY(string_to_array(%s, ','))"
|
||||
|
|
@ -67,6 +64,7 @@ _UPSTREAM_ASKS_FOR_USAGE: Final = {
|
|||
},
|
||||
}
|
||||
_SETTING_VALUES: Final = {"always_off": False, "always_on": True}
|
||||
_STREAMS_PER_WORKER: Final = 6
|
||||
|
||||
|
||||
def _openai_frame(chunk: Mapping[str, JsonValue], response_id: str) -> bytes:
|
||||
|
|
@ -273,20 +271,15 @@ def _stream_or_dropped(client: httpx.Client, key: str) -> httpx.Response | None:
|
|||
return None
|
||||
|
||||
|
||||
def _local_port(client: httpx.Client) -> int:
|
||||
with client.stream("GET", "/health/liveliness") as response:
|
||||
port: Final = int(response.extensions["network_stream"].get_extra_info("client_addr")[1])
|
||||
response.read()
|
||||
assert response.status_code == 200, response.text
|
||||
return port
|
||||
|
||||
|
||||
def _accepted_client_ports(pid: int, proxy_port: int) -> frozenset[int]:
|
||||
return frozenset(
|
||||
connection.raddr.port
|
||||
for connection in psutil.Process(pid).net_connections(kind="tcp")
|
||||
if connection.raddr and connection.laddr.port == proxy_port
|
||||
def _held_streams(
|
||||
stack: ExitStack, pool: ThreadPoolExecutor, url: str, keys: Sequence[str], wire: Wire, held: int
|
||||
) -> tuple[Future[httpx.Response | None], ...]:
|
||||
clients: Final = tuple(stack.enter_context(httpx.Client(base_url=url, timeout=60, trust_env=False)) for _ in keys)
|
||||
futures: Final = tuple(
|
||||
pool.submit(_stream_or_dropped, client, key) for client, key in zip(clients, keys, strict=True)
|
||||
)
|
||||
eventually(wire.received.qsize, lambda size: size >= held, seconds=60)
|
||||
return futures
|
||||
|
||||
|
||||
@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
|
|
@ -296,35 +289,27 @@ def test_worker_killed_mid_stream_leaves_the_sibling_billing_each_stream_once(ga
|
|||
wire: Final = stack.enter_context(wire_server(_peer(gate)))
|
||||
stack.callback(gate.set)
|
||||
config: Final = _owned_config(tmp_path / "worker-kill.yaml", gateway, wire, {})
|
||||
owned: Final = stack.enter_context(owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2))
|
||||
workers: Final = eventually(
|
||||
lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())),
|
||||
lambda pids: len(pids) == 2,
|
||||
seconds=30,
|
||||
owned: Final = stack.enter_context(
|
||||
owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2, healthcheck=WorkerHealthcheck.LENIENT)
|
||||
)
|
||||
victim, survivor = sorted(eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=30))
|
||||
scenario: Final = stack.enter_context(owned.gateway.scenario())
|
||||
proxy_url: Final = owned.gateway.client.base_url
|
||||
clients: Final = tuple(
|
||||
stack.enter_context(httpx.Client(base_url=str(proxy_url), timeout=60, trust_env=False)) for _ in range(12)
|
||||
)
|
||||
keys: Final = tuple(scenario.key(models=[_ADAPTER_MODEL]) for _ in clients)
|
||||
ports: Final = tuple(_local_port(client) for client in clients)
|
||||
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients)))
|
||||
futures: Final = tuple(
|
||||
pool.submit(_stream_or_dropped, client, key) for client, key in zip(clients, keys, strict=True)
|
||||
)
|
||||
eventually(wire.received.qsize, lambda size: size >= len(clients), seconds=60)
|
||||
shares: Final = {pid: _accepted_client_ports(pid, proxy_url.port or 0) & frozenset(ports) for pid in workers}
|
||||
assert sum(map(len, shares.values())) == len(clients), shares
|
||||
victim: Final = min((pid for pid in workers if shares[pid]), key=lambda pid: len(shares[pid]))
|
||||
victim_keys: Final = tuple(scenario.key(models=[_ADAPTER_MODEL]) for _ in range(_STREAMS_PER_WORKER))
|
||||
survivor_keys: Final = tuple(scenario.key(models=[_ADAPTER_MODEL]) for _ in range(_STREAMS_PER_WORKER))
|
||||
proxy_url: Final = str(owned.gateway.client.base_url)
|
||||
pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=2 * _STREAMS_PER_WORKER))
|
||||
with owned.suspended(survivor):
|
||||
on_victim: Final = _held_streams(stack, pool, proxy_url, victim_keys, wire, _STREAMS_PER_WORKER)
|
||||
with owned.suspended(victim):
|
||||
on_survivor: Final = _held_streams(stack, pool, proxy_url, survivor_keys, wire, 2 * _STREAMS_PER_WORKER)
|
||||
with owned.replacing_worker(victim) as kill:
|
||||
kill()
|
||||
gate.set()
|
||||
results: Final = tuple(future.result(timeout=60) for future in futures)
|
||||
survivors: Final = tuple(
|
||||
(key, result) for port, key, result in zip(ports, keys, results, strict=True) if port not in shares[victim]
|
||||
)
|
||||
assert survivors, shares
|
||||
dropped: Final = tuple(future.result(timeout=60) for future in on_victim)
|
||||
survivors: Final = tuple(
|
||||
zip(survivor_keys, (future.result(timeout=60) for future in on_survivor), strict=True)
|
||||
)
|
||||
assert dropped == (None,) * _STREAMS_PER_WORKER, dropped
|
||||
for _, result in survivors:
|
||||
assert result is not None
|
||||
_assert_one_stop_with(result, _GEMINI_USAGE)
|
||||
|
|
|
|||
|
|
@ -379,48 +379,47 @@ def test_a_replacing_block_that_never_kills_its_worker_fails_and_leaves_it_servi
|
|||
assert owned.worker_pids() == frozenset((worker,))
|
||||
|
||||
|
||||
def _kill_of(owned: OwnedProxy, worker: int) -> WorkerKill:
|
||||
return WorkerKill(psutil.Process(worker), psutil.Process(owned.process.pid), 5, threading.Event())
|
||||
@contextmanager
|
||||
def _unreaped_child() -> Iterator[psutil.Process]:
|
||||
with subprocess.Popen((sys.executable, "-I", "-c", "import time; time.sleep(60)")) as child:
|
||||
try:
|
||||
yield psutil.Process(child.pid)
|
||||
finally:
|
||||
child.kill()
|
||||
|
||||
|
||||
def test_a_worker_kill_returns_once_the_supervisor_reaped_the_victim_and_cannot_fire_twice(tmp_path: Path) -> None:
|
||||
with _supervised(tmp_path, WorkerHealthcheck.STRICT) as (owned, worker):
|
||||
kill: Final = _kill_of(owned, worker)
|
||||
def test_a_worker_kill_returns_once_the_victim_is_dead_before_anything_reaps_it_and_cannot_fire_twice() -> None:
|
||||
with _unreaped_child() as victim:
|
||||
kill: Final = WorkerKill(victim, threading.Event())
|
||||
kill()
|
||||
assert not psutil.pid_exists(worker)
|
||||
assert victim.status() == psutil.STATUS_ZOMBIE
|
||||
assert kill.fired.is_set()
|
||||
with pytest.raises(AssertionError, match="already killed"):
|
||||
kill()
|
||||
|
||||
|
||||
def test_a_frozen_victim_stays_stopped_until_the_kill(tmp_path: Path) -> None:
|
||||
with _supervised(tmp_path, WorkerHealthcheck.STRICT) as (owned, worker):
|
||||
kill: Final = _kill_of(owned, worker)
|
||||
def test_a_frozen_victim_stays_stopped_until_the_kill() -> None:
|
||||
with _unreaped_child() as victim:
|
||||
kill: Final = WorkerKill(victim, threading.Event())
|
||||
assert kill.freeze().status() == psutil.STATUS_STOPPED
|
||||
kill()
|
||||
assert not psutil.pid_exists(worker)
|
||||
assert victim.status() == psutil.STATUS_ZOMBIE
|
||||
|
||||
|
||||
def test_a_kill_under_a_suspended_supervisor_returns_with_the_victim_left_unreaped(tmp_path: Path) -> None:
|
||||
with _supervised(tmp_path, WorkerHealthcheck.LENIENT) as (owned, worker):
|
||||
kill: Final = _kill_of(owned, worker)
|
||||
with owned.suspended(owned.process.pid):
|
||||
kill()
|
||||
assert psutil.Process(worker).status() == psutil.STATUS_ZOMBIE
|
||||
eventually(lambda: psutil.pid_exists(worker), lambda exists: not exists, seconds=5)
|
||||
def _kill_after_freezing_the_supervisor(owned: OwnedProxy, worker: int, supervisor: psutil.Process) -> None:
|
||||
with owned.replacing_worker(worker) as kill:
|
||||
supervisor.suspend()
|
||||
kill()
|
||||
|
||||
|
||||
def test_a_kill_fails_when_the_supervisor_never_reaps_the_victim(tmp_path: Path) -> None:
|
||||
def test_a_replacing_block_fails_when_the_supervisor_never_reaps_the_killed_worker(tmp_path: Path) -> None:
|
||||
with _supervised(tmp_path, WorkerHealthcheck.STRICT) as (owned, worker):
|
||||
unreaping: Final = psutil.Process(owned.process.pid)
|
||||
kill: Final = WorkerKill(psutil.Process(worker), psutil.Process(os.getpid()), 0.5, threading.Event())
|
||||
unreaping.suspend()
|
||||
try:
|
||||
with pytest.raises(AssertionError, match="did not reap"):
|
||||
kill()
|
||||
_kill_after_freezing_the_supervisor(owned, worker, unreaping)
|
||||
finally:
|
||||
unreaping.resume()
|
||||
assert not kill.fired.is_set()
|
||||
|
||||
|
||||
def test_a_strict_healthcheck_proxy_refuses_suspension_before_stopping_anything(tmp_path: Path) -> None:
|
||||
|
|
|
|||
|
|
@ -9,12 +9,14 @@ response. A failure raised after the router picked a deployment (a post-call
|
|||
guardrail block, a provider error) carries no flag and keeps its deployment labels.
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from prometheus_client import REGISTRY
|
||||
|
||||
from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY
|
||||
from litellm.exceptions import GuardrailRaisedException
|
||||
from litellm.exceptions import GuardrailRaisedException, MalformedRequestFieldError
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
|
@ -99,3 +101,15 @@ def test_failure_after_routing_still_marks_deployment_partial_outage(failure: Ex
|
|||
|
||||
assert _model_id_values(logger.litellm_deployment_state) == {"dep-1"}
|
||||
assert _model_id_values(logger.litellm_deployment_failure_responses) == {"dep-1"}
|
||||
|
||||
|
||||
def test_malformed_field_rejection_after_routing_leaves_deployment_healthy() -> None:
|
||||
logger: Final = PrometheusLogger()
|
||||
rejection: Final = MalformedRequestFieldError(
|
||||
message="tools: Input should be a valid list", model="internal-model", llm_provider="openai"
|
||||
)
|
||||
|
||||
logger.set_llm_deployment_failure_metrics(_attributed_failure_kwargs(rejection, rejected_before_routing=False))
|
||||
|
||||
assert logger.litellm_deployment_state._metrics == {}
|
||||
assert _model_id_values(logger.litellm_deployment_failure_responses) == {""}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
|
|
@ -49,11 +51,12 @@ def test_web_search_cost_high():
|
|||
)
|
||||
|
||||
|
||||
def test_get_web_search_options_preserves_explicit_none_error():
|
||||
with pytest.raises(TypeError):
|
||||
StandardBuiltInToolCostTracking.get_web_search_options(
|
||||
{"web_search_options": None, "tools": [{"type": "web_search_preview"}]}
|
||||
)
|
||||
def test_get_web_search_options_reads_tools_when_options_are_null() -> None:
|
||||
options: Final = StandardBuiltInToolCostTracking.get_web_search_options(
|
||||
{"web_search_options": None, "tools": [{"type": "web_search_preview"}]}
|
||||
)
|
||||
|
||||
assert options == {"type": "web_search_preview"}
|
||||
|
||||
|
||||
def test_get_web_search_options_accepts_non_list_tool_iterables():
|
||||
|
|
@ -1070,3 +1073,13 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None:
|
|||
usage = Usage(prompt_tokens=1840, completion_tokens=412, total_tokens=2252)
|
||||
|
||||
assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[{"tools": 5}, {"web_search_options": None}, {"web_search_options": 5}],
|
||||
ids=["int_tools", "null_web_search_options", "int_web_search_options"],
|
||||
)
|
||||
def test_built_in_tool_lookups_ignore_malformed_request_values(kwargs: dict[str, object]) -> None:
|
||||
assert StandardBuiltInToolCostTracking.get_web_search_options(kwargs) is None
|
||||
assert StandardBuiltInToolCostTracking.get_file_search_tool_call(kwargs) is None
|
||||
|
|
|
|||
187
tests/unit/litellm_core_utils/test_tool_fields.py
Normal file
187
tests/unit/litellm_core_utils/test_tool_fields.py
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
import datetime
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.utils import async_pre_call_deployment_hook
|
||||
|
||||
_GOOD_TOOL: Final = {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}
|
||||
_MALFORMED_TOOLS: Final = (
|
||||
5,
|
||||
"get_weather",
|
||||
["get_weather"],
|
||||
{"type": "function"},
|
||||
[{"type": "function", "function": {"name": "get_weather", "parameters": "object"}}],
|
||||
)
|
||||
|
||||
|
||||
def _deployment(name: str, **params: object) -> Mapping[str, object]:
|
||||
return {
|
||||
"model_name": name,
|
||||
"litellm_params": {"model": "openai/gpt-5.2", "api_key": "sk-test", "mock_response": f"served by {name}", **params},
|
||||
}
|
||||
|
||||
|
||||
def _proxy_request(model: str, route: str = "acompletion", **fields: object) -> Mapping[str, object]:
|
||||
return {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "weather in Paris?"}],
|
||||
"proxy_server_request": {"url": "/v1/chat/completions", "body": {}},
|
||||
"litellm_logging_obj": Logging(
|
||||
model=model,
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type=route,
|
||||
start_time=datetime.datetime.now(),
|
||||
litellm_call_id="call-1",
|
||||
function_id="fn-1",
|
||||
),
|
||||
"num_retries": 0,
|
||||
**fields,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tools", _MALFORMED_TOOLS)
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_tools_fall_back_to_a_deployment_that_drops_them(tools: object) -> None:
|
||||
router: Final = Router(
|
||||
model_list=[_deployment("primary"), _deployment("backup", additional_drop_params=["tools"])],
|
||||
fallbacks=[{"primary": ["backup"]}],
|
||||
)
|
||||
|
||||
response: Final = await router.acompletion(**_proxy_request("primary", tools=tools))
|
||||
|
||||
assert response.choices[0].message.content == "served by backup"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_on_unsupported_params_false_keeps_the_400() -> None:
|
||||
router: Final = Router(
|
||||
model_list=[_deployment("primary"), _deployment("backup", additional_drop_params=["tools"])],
|
||||
fallbacks=[{"primary": ["backup"]}],
|
||||
fallback_on_unsupported_params=False,
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
await router.acompletion(**_proxy_request("primary", tools=5))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_rejected_request_cools_down_no_deployment() -> None:
|
||||
cools_on_any_400: Final = {"allowed_fails_policy": {"BadRequestErrorAllowedFails": 0}, "cooldown_time": 120}
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{**_deployment("primary"), "model_info": {"id": "primary", **cools_on_any_400}},
|
||||
{**_deployment("backup"), "model_info": {"id": "backup", **cools_on_any_400}},
|
||||
],
|
||||
fallbacks=[{"primary": ["backup"]}],
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
await router.acompletion(**_proxy_request("primary", tools=5))
|
||||
|
||||
primary: Final = await router.acompletion(**_proxy_request("primary"))
|
||||
backup: Final = await router.acompletion(**_proxy_request("backup"))
|
||||
assert (primary.choices[0].message.content, backup.choices[0].message.content) == (
|
||||
"served by primary",
|
||||
"served by backup",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tools", _MALFORMED_TOOLS)
|
||||
@pytest.mark.parametrize(
|
||||
"deployment",
|
||||
[_deployment("keeps"), _deployment("keeps", tools=[_GOOD_TOOL])],
|
||||
ids=["plain", "with_configured_tools"],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_tools_answer_400_when_the_served_deployment_keeps_tools(
|
||||
tools: object, deployment: Mapping[str, object]
|
||||
) -> None:
|
||||
router: Final = Router(model_list=[deployment])
|
||||
|
||||
with pytest.raises(litellm.BadRequestError) as raised:
|
||||
await router.acompletion(**_proxy_request("keeps", tools=tools))
|
||||
|
||||
assert (raised.value.status_code, raised.value.param, raised.value.type) == (
|
||||
400,
|
||||
"tools",
|
||||
"invalid_request_error",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_request_drop_params_false_overrides_a_deployment_that_drops() -> None:
|
||||
router: Final = Router(model_list=[_deployment("drops", drop_params=True)])
|
||||
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
await router.acompletion(**_proxy_request("drops", tools=5, drop_params=False))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"route", ["acompletion", "aresponses", "anthropic_messages", "agenerate_content", "acreate_interaction"]
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_drop_params_drops_every_malformed_tool_field(route: str) -> None:
|
||||
request: Final = _proxy_request(
|
||||
"drops", route, drop_params=True, tools=5, tool_choice="auto", web_search_options="high"
|
||||
)
|
||||
|
||||
sent: Final = await async_pre_call_deployment_hook(dict(request), route)
|
||||
|
||||
assert {"tools", "tool_choice", "web_search_options"}.isdisjoint(sent)
|
||||
assert sent["messages"] == request["messages"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_additional_drop_params_drops_only_the_malformed_tool_entries() -> None:
|
||||
request: Final = _proxy_request("drops", additional_drop_params=["tools"], tools=[_GOOD_TOOL, "get_weather"])
|
||||
|
||||
sent: Final = await async_pre_call_deployment_hook(dict(request), "acompletion")
|
||||
|
||||
assert sent["tools"] == [_GOOD_TOOL]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_sdk_call_keeps_the_tools_it_passed() -> None:
|
||||
tools: Final = (_GOOD_TOOL,)
|
||||
request: Final = {
|
||||
key: value for key, value in _proxy_request("sdk", tools=tools).items() if key != "proxy_server_request"
|
||||
}
|
||||
|
||||
sent: Final = await async_pre_call_deployment_hook(dict(request), "acompletion")
|
||||
|
||||
assert sent["tools"] is tools
|
||||
|
||||
|
||||
@pytest.mark.parametrize("schema", ["", [], False, 0, {}, '{"type": "object"}'])
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
[
|
||||
lambda schema: {"type": "function", "function": {"name": "get_weather", "parameters": schema}},
|
||||
lambda schema: {"type": "function", "name": "get_weather", "parameters": schema},
|
||||
lambda schema: {"name": "get_weather", "input_schema": schema},
|
||||
],
|
||||
ids=["chat", "responses", "messages"],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_and_json_string_schemas_reach_the_deployment(
|
||||
tool: Callable[[object], Mapping[str, object]], schema: object
|
||||
) -> None:
|
||||
tools: Final = [tool(schema)]
|
||||
|
||||
sent: Final = await async_pre_call_deployment_hook(dict(_proxy_request("keeps", tools=tools)), "acompletion")
|
||||
|
||||
assert sent["tools"] is tools
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tools", [(_GOOD_TOOL,), [{"type": "web_search", "function": None}]], ids=["tuple", "null"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_tools_core_accepts_reach_the_deployment(tools: object) -> None:
|
||||
sent: Final = await async_pre_call_deployment_hook(dict(_proxy_request("keeps", tools=tools)), "acompletion")
|
||||
|
||||
assert sent["tools"] is tools
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
|
|
@ -19,6 +20,21 @@ from litellm.types.mcp import MCPTransport
|
|||
from litellm.types.mcp_server.mcp_server_manager import MCPServer
|
||||
|
||||
|
||||
def allow_one_request(monkeypatch: pytest.MonkeyPatch, user_id: str, server_id: str) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
||||
windows: Final = {(user_id, server_id): asyncio.Semaphore(1)}
|
||||
|
||||
async def enforce(user: UserAPIKeyAuth | None, server: MCPServer) -> None:
|
||||
window: Final = windows[(user.user_id if user is not None else None, server.server_id)]
|
||||
if window.locked():
|
||||
raise ProxyRateLimitError(detail=f"{server_id} RPM exceeded")
|
||||
await window.acquire()
|
||||
|
||||
monkeypatch.setattr(proxy_server, "proxy_logging_obj", SimpleNamespace(enforce_mcp_server_rate_limits=enforce))
|
||||
|
||||
|
||||
def context(session: ServerSession, revision: str = "2026-07-28") -> ServerRequestContext:
|
||||
return ServerRequestContext(
|
||||
session=session,
|
||||
|
|
@ -356,3 +372,117 @@ async def test_proxy_route_rejects_legacy_resource_subscription_before_setup(met
|
|||
await getattr(server, method)(context(Mock(spec=ServerSession)), params)
|
||||
finally:
|
||||
server._mcp_proxy_mode.reset(token)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("client_requests", "tools_list_admitted"),
|
||||
[((), True), (("subscriptions/listen",), False), (("resources/subscribe",), False)],
|
||||
)
|
||||
async def test_only_client_initiated_listeners_consume_server_rate_limit(
|
||||
monkeypatch: pytest.MonkeyPatch, client_requests: tuple[str, ...], tools_list_admitted: bool
|
||||
) -> None:
|
||||
from litellm.proxy._experimental.mcp_server import operations
|
||||
|
||||
allow_one_request(monkeypatch, "alice", "upstream")
|
||||
opened: Final = asyncio.Queue[None]()
|
||||
|
||||
async def hold_upstream(*_: object, **__: object) -> None:
|
||||
opened.put_nowait(None)
|
||||
await asyncio.Event().wait()
|
||||
|
||||
client: Final = Mock(spec=MCPClient)
|
||||
client.run_with_session = AsyncMock(side_effect=hold_upstream)
|
||||
server: Final = MCPServer(server_id="upstream", name="upstream", url="http://localhost/mcp", transport="http")
|
||||
target: Final = subscriptions.SubscriptionTarget(server, client, "synthetic-fingerprint")
|
||||
|
||||
async def resolve(_: OperationContext) -> tuple[subscriptions.SubscriptionTarget, ...]:
|
||||
return (target,)
|
||||
|
||||
service: Final = subscriptions.GatewaySubscriptions(resolve_targets=resolve)
|
||||
caller: Final = OperationContext(_caller=UserAPIKeyAuth(user_id="alice"))
|
||||
authorize: Final = AsyncMock(return_value=caller)
|
||||
session: Final = Mock(spec=ServerSession)
|
||||
session.client_params = Mock()
|
||||
legacy: Final = context(session, "2025-11-25")
|
||||
requests: Final = {
|
||||
"subscriptions/listen": lambda: service.listen(
|
||||
context(Mock(spec=ServerSession)),
|
||||
SubscriptionsListenRequestParams(notifications=SubscriptionFilter(tools_list_changed=True)),
|
||||
authorize,
|
||||
"alice",
|
||||
),
|
||||
"resources/subscribe": lambda: service.legacy_resource(legacy, "test://value", subscribe=True),
|
||||
}
|
||||
authorization: Final = subscriptions.SubscriptionAuthorization(caller, authorize, lambda *_: "alice")
|
||||
starts: Final = (lambda: service.legacy_session(legacy, authorization), *(requests[r] for r in client_requests))
|
||||
tasks: Final = tuple(asyncio.create_task(start()) for start in starts)
|
||||
try:
|
||||
for _ in tasks:
|
||||
await asyncio.wait_for(opened.get(), 1)
|
||||
rejection: Final = await operations._mcp_server_rate_limit_rejection(server, caller.user_api_key_auth)
|
||||
assert (rejection is None) is tools_list_admitted
|
||||
finally:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_legacy_listener_catalog_refreshes_do_not_consume_server_rate_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from mcp.types import ListToolsResult, ServerCapabilities, Tool, ToolsCapability
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._experimental.mcp_server import catalog, operations
|
||||
from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListOk
|
||||
|
||||
server: Final = MCPServer(server_id="list-changed", name="list-changed", transport=MCPTransport.http)
|
||||
caller: Final = UserAPIKeyAuth(api_key="list-changed-key", user_id="alice")
|
||||
allow_one_request(monkeypatch, "alice", "list-changed")
|
||||
listed: Final = asyncio.Queue[None]()
|
||||
|
||||
async def fetch_tools(_: MCPServer, **__: object) -> tuple[ListToolsResult, ServerListOk]:
|
||||
listed.put_nowait(None)
|
||||
return ListToolsResult(tools=[Tool(name="add", inputSchema={"type": "object"})]), ServerListOk(tool_count=1)
|
||||
|
||||
monkeypatch.setenv("LITELLM_SALT_KEY", "list-changed-test-key")
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", None)
|
||||
monkeypatch.setattr(operations.global_mcp_server_manager, "registry", {server.server_id: server})
|
||||
monkeypatch.setattr(operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server]))
|
||||
monkeypatch.setattr(catalog, "get_filtered_server_tools", fetch_tools)
|
||||
|
||||
upstream: Final = Mock()
|
||||
upstream.server_capabilities = ServerCapabilities(tools=ToolsCapability(list_changed=True))
|
||||
upstream.protocol_version = "2025-11-25"
|
||||
upstream.send_ping = AsyncMock()
|
||||
|
||||
async def run_with_session(connected, **_: object) -> None:
|
||||
await connected(upstream)
|
||||
|
||||
client: Final = Mock(spec=MCPClient)
|
||||
client.run_with_session = AsyncMock(side_effect=run_with_session)
|
||||
target: Final = subscriptions.SubscriptionTarget(server, client, "synthetic-fingerprint")
|
||||
|
||||
async def resolve(_: OperationContext) -> tuple[subscriptions.SubscriptionTarget, ...]:
|
||||
return (target,)
|
||||
|
||||
initial: Final = operations.prepare_context(user_api_key_auth=caller, mcp_servers=[server.server_id])
|
||||
authorize: Final = AsyncMock(return_value=initial)
|
||||
session: Final = Mock(spec=ServerSession)
|
||||
session.client_params = Mock()
|
||||
service: Final = subscriptions.GatewaySubscriptions(resolve_targets=resolve)
|
||||
listener: Final = asyncio.create_task(
|
||||
service.legacy_session(
|
||||
context(session, "2025-11-25"),
|
||||
subscriptions.SubscriptionAuthorization(initial, authorize, lambda *_: "alice"),
|
||||
)
|
||||
)
|
||||
try:
|
||||
for _ in range(2):
|
||||
await asyncio.wait_for(listed.get(), 5)
|
||||
assert await operations._mcp_server_rate_limit_rejection(server, caller) is None
|
||||
finally:
|
||||
listener.cancel()
|
||||
await asyncio.gather(listener, return_exceptions=True)
|
||||
|
|
|
|||
|
|
@ -5,13 +5,16 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objec
|
|||
import json
|
||||
import logging
|
||||
import unittest
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from litellm import Router
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
||||
|
||||
class TestAnthropicEndpoints(unittest.TestCase):
|
||||
|
|
@ -467,6 +470,151 @@ class TestErrorLogCarriesCallId:
|
|||
assert call_id in record.getMessage()
|
||||
|
||||
|
||||
def _claude_router() -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-sonnet",
|
||||
"litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_key": "sk-test"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
_MALFORMED_TOOL_ENTRY: Final = [{"name": "get_weather", "input_schema": "object"}]
|
||||
_SEVEN_TOKENS: Final = TokenCountResponse(
|
||||
total_tokens=7, request_model="claude-sonnet", model_used="claude-sonnet", tokenizer_type="test"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model", "tools"),
|
||||
[
|
||||
("claude-sonnet", 5),
|
||||
("claude-sonnet", ""),
|
||||
("claude-sonnet", ["get_weather"]),
|
||||
("claude-sonnet", _MALFORMED_TOOL_ENTRY),
|
||||
("team-public-name", 5),
|
||||
],
|
||||
ids=["int", "empty_string", "list_of_strings", "input_schema_string", "unnamed_model_int"],
|
||||
)
|
||||
async def test_count_tokens_answers_400_for_malformed_tools_without_counting(model: str, tools: object) -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
request: Final = MagicMock()
|
||||
request.headers = {}
|
||||
counter: Final = AsyncMock()
|
||||
body: Final = {"model": model, "messages": [{"role": "user", "content": "hi"}], "tools": tools}
|
||||
|
||||
with (
|
||||
patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam
|
||||
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts it is never reached
|
||||
patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route
|
||||
):
|
||||
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
assert response.status_code == 400
|
||||
assert json.loads(response.body)["error"]["type"] == "invalid_request_error"
|
||||
counter.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_tokens_counts_malformed_tool_entries_for_a_model_the_router_cannot_name() -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
request: Final = MagicMock()
|
||||
request.headers = {}
|
||||
counter: Final = AsyncMock(return_value=_SEVEN_TOKENS)
|
||||
body: Final = {"model": "team-public-name", "messages": [{"role": "user", "content": "hi"}], "tools": _MALFORMED_TOOL_ENTRY}
|
||||
|
||||
with (
|
||||
patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam
|
||||
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives
|
||||
patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route
|
||||
):
|
||||
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
assert response == {"input_tokens": 7}
|
||||
assert counter.await_args.kwargs["request"].tools == _MALFORMED_TOOL_ENTRY
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("dropping", "headers"),
|
||||
[
|
||||
({"drop_params": True}, {}),
|
||||
({"additional_drop_params": ["tools"]}, {}),
|
||||
({}, {"user-agent": "claude-cli/2.1.0 (external, cli)"}),
|
||||
],
|
||||
ids=["drop_params", "additional_drop_params", "claude_code_user_agent"],
|
||||
)
|
||||
async def test_count_tokens_drops_malformed_tools_when_drop_params_is_set(
|
||||
dropping: dict[str, object], headers: dict[str, str]
|
||||
) -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
request: Final = MagicMock()
|
||||
request.headers = headers
|
||||
counter: Final = AsyncMock(return_value=_SEVEN_TOKENS)
|
||||
messages: Final = [{"role": "user", "content": "hi"}]
|
||||
body: Final = {"model": "claude-sonnet", "messages": messages, "tools": 5, **dropping}
|
||||
|
||||
with (
|
||||
patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam
|
||||
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives
|
||||
):
|
||||
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
assert response == {"input_tokens": 7}
|
||||
counted: Final = counter.await_args.kwargs["request"]
|
||||
assert counted.tools is None
|
||||
assert counted.messages == messages
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_even_with_fallbacks() -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm import Router
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-sonnet",
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-sonnet-5-5",
|
||||
"api_key": "sk-test",
|
||||
"additional_drop_params": ["tools"],
|
||||
},
|
||||
},
|
||||
{"model_name": "backup", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "sk-test"}},
|
||||
],
|
||||
fallbacks=[{"claude-sonnet": ["backup"]}],
|
||||
)
|
||||
request: Final = MagicMock()
|
||||
request.headers = {}
|
||||
counter: Final = AsyncMock(return_value=_SEVEN_TOKENS)
|
||||
body: Final = {"model": "claude-sonnet", "messages": [{"role": "user", "content": "hi"}], "tools": 5}
|
||||
|
||||
with (
|
||||
patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam
|
||||
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives
|
||||
patch.object(proxy_server, "llm_router", new=router), # test-quality-ok: module global imported at call time; no injection seam on this route
|
||||
):
|
||||
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
assert response == {"input_tokens": 7}
|
||||
assert counter.await_args.kwargs["request"].tools is None
|
||||
|
||||
|
||||
class TestEventLoggingBatchEndpoint:
|
||||
"""Test the stubbed event logging batch endpoint"""
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ This test file tests the Generic Guardrail API implementation,
|
|||
specifically focusing on metadata extraction and passing.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
|
@ -16,6 +18,7 @@ import litellm
|
|||
from litellm import ModelResponse
|
||||
from litellm._version import version as litellm_version
|
||||
from litellm.exceptions import GuardrailRaisedException, Timeout
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, MaskedHTTPStatusError
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
|
||||
GenericGuardrailAPI,
|
||||
|
|
@ -38,6 +41,38 @@ def generic_guardrail():
|
|||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def raw_rows_guardrail():
|
||||
return GenericGuardrailAPI(
|
||||
api_base="https://api.test.guardrail.com",
|
||||
headers={"Authorization": "Bearer test-key"},
|
||||
guardrail_name="test-generic-guardrail-raw-rows",
|
||||
event_hook="pre_call",
|
||||
default_on=True,
|
||||
forward_messages_as_sent=True,
|
||||
)
|
||||
|
||||
|
||||
async def _structured_messages_sent(guardrail: GenericGuardrailAPI, rows: list[dict]) -> list:
|
||||
sent_bodies: Final[list[bytes]] = [] # mutable-ok: records every body the guardrail endpoint received
|
||||
|
||||
async def answer(request: httpx.Request) -> httpx.Response:
|
||||
sent_bodies.append(request.content)
|
||||
return httpx.Response(200, json={"action": "NONE"}, request=request)
|
||||
|
||||
endpoint: Final = AsyncHTTPHandler(transport=httpx.MockTransport(answer))
|
||||
try:
|
||||
with patch.object(guardrail, "async_handler", endpoint):
|
||||
await guardrail.apply_guardrail(
|
||||
inputs={"texts": ["Look up 123-45-6789 for me."], "structured_messages": rows},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
finally:
|
||||
await endpoint.close()
|
||||
return [json.loads(body)["structured_messages"] for body in sent_bodies]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_user_api_key_dict():
|
||||
"""Create a mock UserAPIKeyAuth object"""
|
||||
|
|
@ -638,6 +673,157 @@ class TestStructuredMessagesInResponse:
|
|||
assert returned_rows[1] is tool_call_row
|
||||
assert returned_rows[2] == {"role": "tool", "tool_call_id": "call_1", "content": '{"ssn": "[REDACTED]"}'}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("row", "openai_row"),
|
||||
[
|
||||
(
|
||||
{"role": "user", "content": "Look up 123-45-6789 for me.", "name": "pat"},
|
||||
{"role": "user", "content": "Look up 123-45-6789 for me."},
|
||||
),
|
||||
(
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "lookup", "arguments": "{}"},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAA="}}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": []},
|
||||
),
|
||||
],
|
||||
ids=["extra_key", "tool_call_index", "image_block"],
|
||||
)
|
||||
async def test_rows_reach_the_guardrail_as_openai_chat_rows_by_default(
|
||||
self, generic_guardrail: GenericGuardrailAPI, row, openai_row
|
||||
):
|
||||
assert await _structured_messages_sent(generic_guardrail, [row]) == [[openai_row]]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_returned_row_that_differs_from_the_shown_row_is_written_back_by_default(
|
||||
self, generic_guardrail, mock_request_data_input
|
||||
):
|
||||
sent_row = {"role": "user", "content": "my ssn is 123-45-6789", "name": "pat"}
|
||||
|
||||
def answer_with_the_row_as_the_client_sent_it(url, json, headers, **_kwargs):
|
||||
answer = MagicMock()
|
||||
answer.json.return_value = {
|
||||
"action": "GUARDRAIL_INTERVENED",
|
||||
"texts": ["my ssn is [SSN]"],
|
||||
"structured_messages": [sent_row],
|
||||
}
|
||||
answer.raise_for_status = MagicMock()
|
||||
return answer
|
||||
|
||||
with patch.object(
|
||||
generic_guardrail.async_handler, "post", side_effect=answer_with_the_row_as_the_client_sent_it
|
||||
):
|
||||
guardrailed_inputs = await generic_guardrail.apply_guardrail(
|
||||
inputs={"texts": ["my ssn is 123-45-6789"], "structured_messages": [sent_row]},
|
||||
request_data=mock_request_data_input,
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert guardrailed_inputs["structured_messages"] == [sent_row]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rows_echoed_back_with_one_rewrite_write_back_the_rewrite_only(
|
||||
self, raw_rows_guardrail, mock_request_data_input
|
||||
):
|
||||
tool_call_row = {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}, "index": 0}
|
||||
],
|
||||
}
|
||||
original_rows = [
|
||||
{"role": "user", "content": "Look up 123-45-6789 for me.", "name": "pat"},
|
||||
tool_call_row,
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": '{"ssn": "123-45-6789"}'},
|
||||
]
|
||||
|
||||
def echo_with_tool_output_redacted(url, json, headers, **_kwargs):
|
||||
shown_rows = json["structured_messages"]
|
||||
answer = MagicMock()
|
||||
answer.json.return_value = {
|
||||
"action": "GUARDRAIL_INTERVENED",
|
||||
"texts": ["Look up 123-45-6789 for me."],
|
||||
"structured_messages": [
|
||||
shown_rows[0],
|
||||
shown_rows[1],
|
||||
{**shown_rows[2], "content": '{"ssn": "[REDACTED]"}'},
|
||||
],
|
||||
}
|
||||
answer.raise_for_status = MagicMock()
|
||||
return answer
|
||||
|
||||
with patch.object(raw_rows_guardrail.async_handler, "post", side_effect=echo_with_tool_output_redacted):
|
||||
guardrailed_inputs = await raw_rows_guardrail.apply_guardrail(
|
||||
inputs={"texts": ["Look up 123-45-6789 for me."], "structured_messages": original_rows},
|
||||
request_data=mock_request_data_input,
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
returned_rows = guardrailed_inputs["structured_messages"]
|
||||
assert returned_rows[0] == original_rows[0]
|
||||
assert returned_rows[1] == tool_call_row
|
||||
assert returned_rows[2] == {"role": "tool", "tool_call_id": "call_1", "content": '{"ssn": "[REDACTED]"}'}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"row",
|
||||
[
|
||||
{"role": "user", "content": "Look up 123-45-6789 for me.", "name": "pat"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}, "index": 0}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "123-45-6789"}]},
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "lookup", "input": {}}]},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAA="}}],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "ssn"}}
|
||||
],
|
||||
},
|
||||
],
|
||||
ids=["extra_key", "tool_call_index", "tool_result_block", "tool_use_block", "image_block", "document_block"],
|
||||
)
|
||||
async def test_rows_reach_the_guardrail_verbatim_when_forwarded_as_sent(
|
||||
self, raw_rows_guardrail: GenericGuardrailAPI, row
|
||||
):
|
||||
"""Rows are the provider's own shape, so the guardrail must see every key and
|
||||
content block as sent, not a copy filtered through the chat completion types."""
|
||||
assert await _structured_messages_sent(raw_rows_guardrail, [row]) == [[row]]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rows_all_echoed_back_as_shown_leave_the_rewrite_to_texts(
|
||||
self, generic_guardrail, mock_request_data_input
|
||||
|
|
@ -672,6 +858,126 @@ class TestStructuredMessagesInResponse:
|
|||
assert "structured_messages" not in guardrailed_inputs
|
||||
assert guardrailed_inputs["texts"] == ["Never repeat an SSN.", "Look up [REDACTED] for me."]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rows_echoed_without_the_keys_the_server_does_not_know_leave_the_rewrite_to_texts(
|
||||
self, raw_rows_guardrail, mock_request_data_input
|
||||
):
|
||||
"""A server that parses rows with its own message types echoes them back without
|
||||
the keys those types lack, and its texts redaction must still reach the LLM."""
|
||||
original_rows = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}, "index": 0}
|
||||
],
|
||||
"refusal": None,
|
||||
"annotations": [],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "found"},
|
||||
{"role": "user", "content": "my ssn is 123-45-6789"},
|
||||
]
|
||||
|
||||
def echo_known_keys_and_redact_texts(url, json, headers, **_kwargs):
|
||||
rows = json["structured_messages"]
|
||||
answer = MagicMock()
|
||||
answer.json.return_value = {
|
||||
"action": "GUARDRAIL_INTERVENED",
|
||||
"texts": [text.replace("123-45-6789", "[SSN]") for text in json["texts"]],
|
||||
"structured_messages": [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"refusal": None,
|
||||
"tool_calls": [
|
||||
{key: value for key, value in rows[0]["tool_calls"][0].items() if key != "index"}
|
||||
],
|
||||
},
|
||||
rows[1],
|
||||
rows[2],
|
||||
],
|
||||
}
|
||||
answer.raise_for_status = MagicMock()
|
||||
return answer
|
||||
|
||||
with patch.object(raw_rows_guardrail.async_handler, "post", side_effect=echo_known_keys_and_redact_texts):
|
||||
guardrailed_inputs = await raw_rows_guardrail.apply_guardrail(
|
||||
inputs={"texts": ["found", "my ssn is 123-45-6789"], "structured_messages": original_rows},
|
||||
request_data=mock_request_data_input,
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert "structured_messages" not in guardrailed_inputs
|
||||
assert guardrailed_inputs["texts"] == ["found", "my ssn is [SSN]"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_row_rewrite_echoes_the_other_rows_back_as_their_original_rows(
|
||||
self, raw_rows_guardrail, mock_request_data_input
|
||||
):
|
||||
assistant_row = {
|
||||
"role": "assistant",
|
||||
"content": "Looking it up.",
|
||||
"annotations": [{"type": "url_citation", "url_citation": {"url": "https://example.com"}}],
|
||||
}
|
||||
original_rows = [assistant_row, {"role": "user", "content": "my ssn is 123-45-6789"}]
|
||||
redacted_row = {"role": "user", "content": "my ssn is [SSN]"}
|
||||
|
||||
def redact_one_row(url, json, headers, **_kwargs):
|
||||
answer = MagicMock()
|
||||
answer.json.return_value = {
|
||||
"action": "GUARDRAIL_INTERVENED",
|
||||
"structured_messages": [{"role": "assistant", "content": "Looking it up."}, redacted_row],
|
||||
}
|
||||
answer.raise_for_status = MagicMock()
|
||||
return answer
|
||||
|
||||
with patch.object(raw_rows_guardrail.async_handler, "post", side_effect=redact_one_row):
|
||||
guardrailed_inputs = await raw_rows_guardrail.apply_guardrail(
|
||||
inputs={"texts": ["my ssn is 123-45-6789"], "structured_messages": original_rows},
|
||||
request_data=mock_request_data_input,
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert guardrailed_inputs["structured_messages"] == [assistant_row, redacted_row]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"returned_row",
|
||||
[
|
||||
{"role": "assistant", "content": None},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
|
||||
"annotations": [],
|
||||
"refusal": "blocked",
|
||||
},
|
||||
],
|
||||
ids=["drops_a_declared_key", "changes_a_value"],
|
||||
)
|
||||
async def test_a_row_that_loses_a_declared_key_or_changes_a_value_is_a_rewrite(
|
||||
self, raw_rows_guardrail, mock_request_data_input, returned_row
|
||||
):
|
||||
original_row = {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
|
||||
"annotations": [],
|
||||
"refusal": None,
|
||||
}
|
||||
answer = MagicMock()
|
||||
answer.json.return_value = {"action": "GUARDRAIL_INTERVENED", "structured_messages": [returned_row]}
|
||||
answer.raise_for_status = MagicMock()
|
||||
|
||||
with patch.object(raw_rows_guardrail.async_handler, "post", return_value=answer):
|
||||
guardrailed_inputs = await raw_rows_guardrail.apply_guardrail(
|
||||
inputs={"texts": [], "structured_messages": [original_row]},
|
||||
request_data=mock_request_data_input,
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert guardrailed_inputs["structured_messages"] == [returned_row]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"structured_messages",
|
||||
|
|
@ -1253,6 +1559,40 @@ class TestGenericGuardrailAPIStreamingConfig:
|
|||
)
|
||||
assert guardrail.streaming_transform_mode == "incremental_diff"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "forwards_as_sent"),
|
||||
[
|
||||
pytest.param({}, False, id="unset"),
|
||||
pytest.param({"forward_messages_as_sent": ""}, False, id="blank"),
|
||||
pytest.param({"forward_messages_as_sent": False}, False, id="false"),
|
||||
pytest.param({"forward_messages_as_sent": "true"}, True, id="string-true"),
|
||||
pytest.param({"forward_messages_as_sent": True}, True, id="true"),
|
||||
],
|
||||
)
|
||||
async def test_forward_messages_as_sent_config_decides_which_rows_reach_the_guardrail(
|
||||
self, configured, forwards_as_sent
|
||||
):
|
||||
from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
|
||||
initialize_guardrail,
|
||||
)
|
||||
from litellm.types.guardrails import LitellmParams
|
||||
|
||||
litellm_params = LitellmParams.model_validate(
|
||||
{
|
||||
"guardrail": "generic_guardrail_api",
|
||||
"mode": "pre_call",
|
||||
"api_base": "https://api.test.guardrail.com",
|
||||
**configured,
|
||||
}
|
||||
)
|
||||
with patch("litellm.logging_callback_manager.add_litellm_callback"):
|
||||
guardrail = initialize_guardrail(litellm_params, {"guardrail_name": "test-forward-as-sent"})
|
||||
|
||||
row = {"role": "user", "content": "hi", "name": "pat"}
|
||||
expected_row = row if forwards_as_sent else {"role": "user", "content": "hi"}
|
||||
assert await _structured_messages_sent(guardrail, [row]) == [[expected_row]]
|
||||
|
||||
def test_initialize_guardrail_forwards_streaming_flags(self):
|
||||
from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
|
||||
initialize_guardrail,
|
||||
|
|
@ -1945,6 +2285,34 @@ class TestToolSupport:
|
|||
|
||||
assert forwarded_tools == tools
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_provider_native_tools_without_a_type_reach_the_guardrail_and_come_back_unchanged(
|
||||
self, generic_guardrail
|
||||
):
|
||||
"""Gemini tools are declared as function_declarations, googleSearch and the
|
||||
like with no type key, so they must be scanned, and a server that echoes
|
||||
them back must not write a type key into the request to the provider."""
|
||||
tools = [
|
||||
{"function_declarations": [{"name": "get_weather", "parameters": {"type": "object", "properties": {}}}]},
|
||||
{"googleSearch": {}},
|
||||
]
|
||||
|
||||
def echo_tools(url, json, headers, **_kwargs):
|
||||
answer = MagicMock()
|
||||
answer.json.return_value = {"action": "NONE", "texts": ["hi"], "tools": json["tools"]}
|
||||
answer.raise_for_status = MagicMock()
|
||||
return answer
|
||||
|
||||
with patch.object(generic_guardrail.async_handler, "post", side_effect=echo_tools) as mock_post:
|
||||
guardrailed_inputs = await generic_guardrail.apply_guardrail(
|
||||
inputs={"texts": ["hi"], "tools": tools},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
|
||||
assert mock_post.call_args.kwargs["json"]["tools"] == tools
|
||||
assert guardrailed_inputs["tools"] == tools
|
||||
|
||||
|
||||
class TestFailOnError:
|
||||
"""Test fail_on_error: complete fail-open on any guardrail error"""
|
||||
|
|
@ -2032,6 +2400,59 @@ class TestFailOnError:
|
|||
|
||||
assert result == {"texts": ["model output"]}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("error", "error_type", "pre_pr_text"),
|
||||
[
|
||||
(
|
||||
Timeout(message="guardrail took too long", model="gpt-5", llm_provider="openai"),
|
||||
"Timeout",
|
||||
"Proceeding without guardrail. guardrail_name=",
|
||||
),
|
||||
(
|
||||
httpx.ConnectError("connection refused", request=MagicMock()),
|
||||
"ConnectError",
|
||||
"Proceeding without guardrail. guardrail_name=",
|
||||
),
|
||||
(
|
||||
MaskedHTTPStatusError(
|
||||
httpx.HTTPStatusError(
|
||||
"bad gateway",
|
||||
request=httpx.Request("POST", "http://guardrail.invalid/scan"),
|
||||
response=httpx.Response(502, request=httpx.Request("POST", "http://guardrail.invalid/scan")),
|
||||
)
|
||||
),
|
||||
"HTTPStatusError",
|
||||
"Proceeding without guardrail. http_status_code=502 guardrail_name=",
|
||||
),
|
||||
],
|
||||
ids=["timeout", "connect_error", "http_status"],
|
||||
)
|
||||
async def test_fail_open_log_line_names_the_error_type(
|
||||
self, fail_open_guardrail, caplog: pytest.LogCaptureFixture, error, error_type, pre_pr_text
|
||||
):
|
||||
"""An operator grepping the fail-open line must be able to tell a timeout
|
||||
from a refused connection from an upstream status without the traceback."""
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
verbose_proxy_logger.addHandler(caplog.handler)
|
||||
try:
|
||||
with patch.object(fail_open_guardrail.async_handler, "post", side_effect=error):
|
||||
await fail_open_guardrail.apply_guardrail(
|
||||
inputs={"texts": ["hi"]},
|
||||
request_data={},
|
||||
input_type="request",
|
||||
)
|
||||
finally:
|
||||
verbose_proxy_logger.removeHandler(caplog.handler)
|
||||
|
||||
fail_open_lines: Final = [
|
||||
record.getMessage() for record in caplog.records if record.levelno == logging.CRITICAL
|
||||
]
|
||||
assert fail_open_lines
|
||||
assert all(f"error_type={error_type}" in line.split() for line in fail_open_lines)
|
||||
assert all(pre_pr_text in line for line in fail_open_lines)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_path_valid_block_still_blocks(self, fail_open_guardrail):
|
||||
"""On the response path too, a valid BLOCKED decision raises despite fail_on_error=False."""
|
||||
|
|
|
|||
|
|
@ -953,6 +953,17 @@ async def test_provider_specific_params_includes_embedding_toggle():
|
|||
assert field["default_value"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_provider_specific_params_offers_forward_messages_as_sent_off_by_default():
|
||||
from litellm.proxy.guardrails.guardrail_endpoints import get_provider_specific_params
|
||||
|
||||
provider_params = await get_provider_specific_params()
|
||||
|
||||
field = provider_params["generic_guardrail_api"]["optional_params"]["fields"]["forward_messages_as_sent"]
|
||||
assert field["type"] == "boolean"
|
||||
assert field["default_value"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_provider_specific_params_exposes_bedrock_streaming_flags():
|
||||
from litellm.proxy.guardrails.guardrail_endpoints import get_provider_specific_params
|
||||
|
|
|
|||
|
|
@ -201,6 +201,10 @@ class RecordingObserver(PromptCacheObserver):
|
|||
await super().async_log_success_event(kwargs, response_obj, start_time, end_time)
|
||||
self.finished.set()
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
await super().async_log_failure_event(kwargs, response_obj, start_time, end_time)
|
||||
self.finished.set()
|
||||
|
||||
|
||||
def native_response():
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import asyncio
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from contextlib import ExitStack, asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
|
|
@ -17,7 +17,7 @@ from fastapi import HTTPException, Request
|
|||
import litellm
|
||||
from litellm._uuid import uuid
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.proxy._types import LiteLLM_UserTable, NewUserResponse
|
||||
from litellm.proxy._types import LiteLLM_UserTable, NewUserResponse, ProxyException
|
||||
from litellm.proxy.auth.handle_jwt import JWTHandler
|
||||
from litellm.proxy.management_endpoints.sso import CustomMicrosoftSSO
|
||||
from litellm.proxy.management_endpoints.types import CustomOpenID
|
||||
|
|
@ -26,9 +26,11 @@ from litellm.proxy.management_endpoints.ui_sso import (
|
|||
MicrosoftSSOHandler,
|
||||
SSOAuthenticationHandler,
|
||||
_complete_cli_sso_callback_session,
|
||||
_insert_sso_user_or_find_concurrent_insert,
|
||||
_setup_team_mappings,
|
||||
_sync_user_role_from_jwt_role_map,
|
||||
cli_sso_callback,
|
||||
insert_sso_user,
|
||||
normalize_email,
|
||||
process_sso_jwt_access_token,
|
||||
sso_lookup_user_ids,
|
||||
|
|
@ -1425,6 +1427,82 @@ async def test_upsert_sso_user_no_role_in_sso_response():
|
|||
assert "user_role" not in call_args.kwargs["data"]
|
||||
|
||||
|
||||
class _UserTable:
|
||||
def __init__(self, *rows: LiteLLM_UserTable) -> None:
|
||||
self.rows: dict[str, LiteLLM_UserTable] = {row.user_id: row for row in rows}
|
||||
|
||||
async def find_first(self, where: dict[str, dict[str, str]]) -> LiteLLM_UserTable | None:
|
||||
((field, condition),) = where.items()
|
||||
|
||||
def matches(stored: object) -> bool:
|
||||
if condition.get("mode") == "insensitive":
|
||||
return str(stored).lower() == condition["equals"].lower()
|
||||
return stored == condition["equals"]
|
||||
|
||||
return next((row for row in self.rows.values() if matches(getattr(row, field))), None)
|
||||
|
||||
async def find_unique(self, where: dict[str, str]) -> LiteLLM_UserTable | None:
|
||||
return self.rows.get(where["user_id"])
|
||||
|
||||
|
||||
def _users_db(table: _UserTable) -> SimpleNamespace:
|
||||
database: Final = SimpleNamespace(litellm_usertable=table)
|
||||
return SimpleNamespace(db=database, writer_db=database)
|
||||
|
||||
|
||||
async def _first_sign_in(table: _UserTable, **join: object) -> NewUserResponse | LiteLLM_UserTable:
|
||||
prisma: Final = _users_db(table)
|
||||
with patch("litellm.proxy.proxy_server.prisma_client", prisma):
|
||||
return await _insert_sso_user_or_find_concurrent_insert(
|
||||
insert=insert_sso_user(
|
||||
result_openid=CustomOpenID(id="sso-subject", email="person@example.com", team_ids=[]),
|
||||
user_defined_values={
|
||||
"user_id": "sso-subject",
|
||||
"user_email": "person@example.com",
|
||||
"user_role": None,
|
||||
"max_budget": None,
|
||||
"budget_duration": None,
|
||||
"models": [],
|
||||
},
|
||||
),
|
||||
user_id="sso-subject",
|
||||
prisma_client=prisma,
|
||||
**join,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_sign_in_signs_in_to_the_account_a_concurrent_first_sign_in_created():
|
||||
winner: Final = LiteLLM_UserTable(user_id="sso-subject", user_email="person@example.com", user_role="internal_user")
|
||||
|
||||
signed_in: Final = await _first_sign_in(_UserTable(winner))
|
||||
|
||||
assert signed_in == winner
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_sign_in_does_not_sign_in_to_another_account_holding_the_email():
|
||||
other: Final = LiteLLM_UserTable(user_id="someone-else", user_email="Person@example.com", user_role="proxy_admin")
|
||||
|
||||
with pytest.raises(ProxyException, match=r"User with email Person@example\.com already exists"):
|
||||
await _first_sign_in(_UserTable(other))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_sign_in_joins_default_teams_the_concurrent_first_sign_in_has_not_joined_yet(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "default_internal_user_params", {"teams": ["team-joined", "team-pending"]})
|
||||
table: Final = _UserTable(
|
||||
LiteLLM_UserTable(user_id="sso-subject", user_email="person@example.com", teams=["team-joined"])
|
||||
)
|
||||
|
||||
async def join(user: LiteLLM_UserTable, teams: Sequence[object], prisma_client: object) -> None:
|
||||
table.rows[user.user_id] = user.model_copy(update={"teams": [*user.teams, *map(str, teams)]})
|
||||
|
||||
signed_in: Final = await _first_sign_in(table, join_default_teams=join)
|
||||
|
||||
assert signed_in.teams == ["team-joined", "team-pending"]
|
||||
|
||||
|
||||
def test_get_user_email_and_id_extracts_microsoft_role():
|
||||
"""
|
||||
Test that _get_user_email_and_id_from_result extracts user_role from Microsoft SSO.
|
||||
|
|
|
|||
|
|
@ -5,11 +5,10 @@ from typing import Any, Dict, Final, List
|
|||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
import litellm
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.pass_through.stream_assembly import extract_sse_data
|
||||
|
|
@ -2644,6 +2643,58 @@ class TestRecordPartialUsageForFailure:
|
|||
assert "response_cost" not in logging_obj.model_call_details
|
||||
|
||||
|
||||
class _FailureRecorder(CustomLogger):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.failure_kwargs = []
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
self.failure_kwargs.append(kwargs)
|
||||
|
||||
|
||||
class TestLogErrorFrameAsFailure:
|
||||
"""A stream closed at the provider's `event: error` frame is one failed attempt: its failure log bills the
|
||||
usage streamed before the frame on the attempt-scoped copy its caller hands it, and the request's shared
|
||||
logging state stays as it was, so the attempt the router opens next logs with nothing left behind."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logs_a_failure_with_the_usage_streamed_before_the_frame_and_leaves_the_request_state_alone(self):
|
||||
recorder = _FailureRecorder()
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
stream=True,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-error-frame-failure",
|
||||
function_id="test-error-frame-failure",
|
||||
dynamic_async_failure_callbacks=[recorder],
|
||||
)
|
||||
chunks = [
|
||||
*TestRecordPartialUsageForFailure()._interrupted_chunks()[:1],
|
||||
TestRecordPartialUsageForFailure._sse(
|
||||
"error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}
|
||||
),
|
||||
]
|
||||
|
||||
await AnthropicPassthroughLoggingHandler.log_error_frame_as_failure(
|
||||
attempt_logging_obj=logging_obj.attempt_scoped_copy(),
|
||||
request_body={"model": "claude-sonnet-5", "stream": True},
|
||||
all_chunks=chunks,
|
||||
error_event=("overloaded_error", "Overloaded", 529),
|
||||
)
|
||||
|
||||
assert len(recorder.failure_kwargs) == 1
|
||||
failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
|
||||
assert failure_payload["status"] == "failure"
|
||||
assert failure_payload["prompt_tokens"] == 52
|
||||
assert failure_payload["response_cost"] > 0
|
||||
assert isinstance(recorder.failure_kwargs[0]["exception"], litellm.InternalServerError)
|
||||
assert "combined_usage_object" not in logging_obj.model_call_details
|
||||
assert "response_cost" not in logging_obj.model_call_details
|
||||
assert logging_obj.should_run_logging("async_failure")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("all_chunks", "interrupted"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthr
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import aclose_superseded_attempt
|
||||
from litellm.proxy.pass_through_endpoints.streaming_handler import (
|
||||
PassThroughStreamingHandler,
|
||||
)
|
||||
|
|
@ -749,6 +750,294 @@ async def test_chunk_processor_logs_failure_not_success_on_mid_stream_exception(
|
|||
assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout)
|
||||
|
||||
|
||||
def _anthropic_message_start_frame(message_id: str = "msg_1", input_tokens: int = 52) -> bytes:
|
||||
return _anthropic_sse(
|
||||
"message_start",
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": message_id,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5",
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": input_tokens, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_completed_stream_tail() -> tuple[bytes, ...]:
|
||||
return (
|
||||
_anthropic_sse(
|
||||
"content_block_start",
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
),
|
||||
_anthropic_sse(
|
||||
"content_block_delta",
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pong"}},
|
||||
),
|
||||
_anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}),
|
||||
_anthropic_sse(
|
||||
"message_delta",
|
||||
{"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 3}},
|
||||
),
|
||||
_anthropic_sse("message_stop", {"type": "message_stop"}),
|
||||
)
|
||||
|
||||
|
||||
async def _settle(recorded: list[dict[str, object]]) -> None:
|
||||
for _ in range(300):
|
||||
if recorded:
|
||||
return
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
def _anthropic_overloaded_error_frame() -> bytes:
|
||||
return _anthropic_sse("error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}})
|
||||
|
||||
|
||||
def _anthropic_stream_of(frames: tuple[bytes, ...]):
|
||||
mock = MagicMock(spec=httpx.Response)
|
||||
mock.status_code = 200
|
||||
|
||||
async def _aiter_bytes():
|
||||
for frame in frames:
|
||||
yield frame
|
||||
|
||||
mock.aiter_bytes = _aiter_bytes
|
||||
return mock
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"frames, billed",
|
||||
[
|
||||
pytest.param(
|
||||
(_anthropic_message_start_frame(), _anthropic_overloaded_error_frame()), True, id="error-after-message-start"
|
||||
),
|
||||
pytest.param((_anthropic_overloaded_error_frame(),), False, id="error-frame-only"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_chunk_processor_logs_a_provider_error_frame_as_a_failure_when_closed_at_the_frame(frames, billed):
|
||||
"""The router retries a /v1/messages stream whose provider sent `event: error` before any content by closing
|
||||
it at the frame. That attempt never completed: it logs once as a failure carrying whatever usage the provider
|
||||
had reported, and the request's one-shot success log stays free for the retry that shares the logging object."""
|
||||
recorder = _EventRecorder()
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-error-frame",
|
||||
function_id="test-error-frame",
|
||||
dynamic_async_success_callbacks=[recorder],
|
||||
dynamic_async_failure_callbacks=[recorder],
|
||||
)
|
||||
stream = PassThroughStreamingHandler.chunk_processor(
|
||||
response=_anthropic_stream_of(frames),
|
||||
request_body={"model": "claude-sonnet-5", "stream": True},
|
||||
litellm_logging_obj=logging_obj,
|
||||
endpoint_type=EndpointType.ANTHROPIC,
|
||||
start_time=datetime.now(),
|
||||
passthrough_success_handler_obj=MagicMock(),
|
||||
url_route="/v1/messages",
|
||||
)
|
||||
received = []
|
||||
async for chunk in stream:
|
||||
received.append(chunk)
|
||||
if b"event: error" in chunk:
|
||||
break
|
||||
await aclose_superseded_attempt(stream)
|
||||
await _settle(recorder.failure_kwargs)
|
||||
|
||||
assert received == list(frames)
|
||||
assert recorder.success_kwargs == []
|
||||
assert logging_obj.model_call_details.get("has_dispatched_final_stream_success") is not True
|
||||
assert "combined_usage_object" not in logging_obj.model_call_details
|
||||
assert "response_cost" not in logging_obj.model_call_details
|
||||
assert logging_obj.should_run_logging("async_failure")
|
||||
assert len(recorder.failure_kwargs) == 1
|
||||
failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
|
||||
assert failure_payload["status"] == "failure"
|
||||
assert isinstance(recorder.failure_kwargs[0]["exception"], litellm.InternalServerError)
|
||||
if billed:
|
||||
assert failure_payload["prompt_tokens"] == 52
|
||||
assert failure_payload["response_cost"] > 0
|
||||
else:
|
||||
assert failure_payload["response_cost"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chunk_processor_logs_a_stream_that_ended_at_the_provider_error_frame_through_the_success_route():
|
||||
"""A provider error frame that ends a stream nobody closed early is the request's final answer, and that
|
||||
stream keeps logging the way it always has: once, through the success route, with the usage it carried."""
|
||||
recorder = _EventRecorder()
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-final-error-frame",
|
||||
function_id="test-final-error-frame",
|
||||
dynamic_async_success_callbacks=[recorder],
|
||||
dynamic_async_failure_callbacks=[recorder],
|
||||
)
|
||||
frames = (_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())
|
||||
received = [
|
||||
chunk
|
||||
async for chunk in PassThroughStreamingHandler.chunk_processor(
|
||||
response=_anthropic_stream_of(frames),
|
||||
request_body={"model": "claude-sonnet-5", "stream": True},
|
||||
litellm_logging_obj=logging_obj,
|
||||
endpoint_type=EndpointType.ANTHROPIC,
|
||||
start_time=datetime.now(),
|
||||
passthrough_success_handler_obj=MagicMock(),
|
||||
url_route="/v1/messages",
|
||||
)
|
||||
]
|
||||
await _settle(recorder.success_kwargs)
|
||||
|
||||
assert received == list(frames)
|
||||
assert recorder.failure_kwargs == []
|
||||
success_payload = recorder.success_kwargs[0]["standard_logging_object"]
|
||||
assert success_payload["status"] == "success"
|
||||
assert success_payload["prompt_tokens"] == 52
|
||||
assert logging_obj.model_call_details["prompt_cache_response_complete"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chunk_processor_bills_a_stream_the_client_closed_at_the_provider_error_frame():
|
||||
"""The Anthropic SDK raises on a provider `event: error` frame and drops the connection, so a non-retried
|
||||
stream is closed at the frame by the client, not by a retry taking over. Its usage must still reach the
|
||||
success route, or the tokens the provider billed never reach the key's spend."""
|
||||
recorder = _EventRecorder()
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-client-closed-at-error-frame",
|
||||
function_id="test-client-closed-at-error-frame",
|
||||
dynamic_async_success_callbacks=[recorder],
|
||||
dynamic_async_failure_callbacks=[recorder],
|
||||
)
|
||||
stream = PassThroughStreamingHandler.chunk_processor(
|
||||
response=_anthropic_stream_of((_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())),
|
||||
request_body={"model": "claude-sonnet-5", "stream": True},
|
||||
litellm_logging_obj=logging_obj,
|
||||
endpoint_type=EndpointType.ANTHROPIC,
|
||||
start_time=datetime.now(),
|
||||
passthrough_success_handler_obj=MagicMock(),
|
||||
url_route="/v1/messages",
|
||||
)
|
||||
async for chunk in stream:
|
||||
if b"event: error" in chunk:
|
||||
break
|
||||
await stream.aclose()
|
||||
await _settle(recorder.success_kwargs)
|
||||
|
||||
assert recorder.failure_kwargs == []
|
||||
success_payload = recorder.success_kwargs[0]["standard_logging_object"]
|
||||
assert success_payload["status"] == "success"
|
||||
assert success_payload["prompt_tokens"] == 52
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_superseded_attempt_failure_log_keeps_its_own_deployment_when_the_retry_rebinds_the_logging_object():
|
||||
"""The router rebinds the shared logging object to the retry's deployment right after closing the superseded
|
||||
stream, before the logging worker runs. The failure log of the superseded attempt must still name the
|
||||
deployment that served it, or the router's cooldown lands on the deployment serving the retry."""
|
||||
recorder = _EventRecorder()
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-superseded-deployment",
|
||||
function_id="test-superseded-deployment",
|
||||
dynamic_async_failure_callbacks=[recorder],
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model="claude-sonnet-5", litellm_params={"model_info": {"id": "deployment-1"}}, optional_params={}
|
||||
)
|
||||
superseded = PassThroughStreamingHandler.chunk_processor(
|
||||
response=_anthropic_stream_of((_anthropic_message_start_frame(), _anthropic_overloaded_error_frame())),
|
||||
request_body={"model": "claude-sonnet-5", "stream": True},
|
||||
litellm_logging_obj=logging_obj,
|
||||
endpoint_type=EndpointType.ANTHROPIC,
|
||||
start_time=datetime.now(),
|
||||
passthrough_success_handler_obj=MagicMock(),
|
||||
url_route="/v1/messages",
|
||||
)
|
||||
async for chunk in superseded:
|
||||
if b"event: error" in chunk:
|
||||
break
|
||||
await aclose_superseded_attempt(superseded)
|
||||
logging_obj.update_environment_variables(
|
||||
model="claude-sonnet-5-retry", litellm_params={"model_info": {"id": "deployment-2"}}, optional_params={}
|
||||
)
|
||||
await _settle(recorder.failure_kwargs)
|
||||
|
||||
failure_kwargs = recorder.failure_kwargs[0]
|
||||
assert failure_kwargs["litellm_params"]["model_info"]["id"] == "deployment-1"
|
||||
assert failure_kwargs["model"] == "claude-sonnet-5"
|
||||
assert logging_obj.model_call_details["litellm_params"]["model_info"]["id"] == "deployment-2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_success_log_carries_its_own_usage_after_a_provider_error_frame():
|
||||
"""Both attempts of a retried /v1/messages stream log through the request's one logging object. The
|
||||
superseded attempt's failure log must not leave the usage it consumed behind, or the retry's success
|
||||
log bills the failed attempt's tokens instead of its own."""
|
||||
recorder = _EventRecorder()
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="test-retry-own-usage",
|
||||
function_id="test-retry-own-usage",
|
||||
dynamic_async_success_callbacks=[recorder],
|
||||
dynamic_async_failure_callbacks=[recorder],
|
||||
)
|
||||
|
||||
def attempt(frames: tuple[bytes, ...]):
|
||||
return PassThroughStreamingHandler.chunk_processor(
|
||||
response=_anthropic_stream_of(frames),
|
||||
request_body={"model": "claude-sonnet-5", "stream": True},
|
||||
litellm_logging_obj=logging_obj,
|
||||
endpoint_type=EndpointType.ANTHROPIC,
|
||||
start_time=datetime.now(),
|
||||
passthrough_success_handler_obj=MagicMock(),
|
||||
url_route="/v1/messages",
|
||||
)
|
||||
|
||||
superseded = attempt((_anthropic_message_start_frame("msg_a1", 52), _anthropic_overloaded_error_frame()))
|
||||
async for chunk in superseded:
|
||||
if b"event: error" in chunk:
|
||||
break
|
||||
await aclose_superseded_attempt(superseded)
|
||||
await _settle(recorder.failure_kwargs)
|
||||
retry = attempt((_anthropic_message_start_frame("msg_a2", 7), *_anthropic_completed_stream_tail()))
|
||||
async for _ in retry:
|
||||
pass
|
||||
await _settle(recorder.success_kwargs)
|
||||
|
||||
failure_payload = recorder.failure_kwargs[0]["standard_logging_object"]
|
||||
assert failure_payload["prompt_tokens"] == 52
|
||||
success_payload = recorder.success_kwargs[0]["standard_logging_object"]
|
||||
assert success_payload["status"] == "success"
|
||||
assert (success_payload["prompt_tokens"], success_payload["completion_tokens"]) == (7, 3)
|
||||
|
||||
|
||||
def _google_sse(prompt_tokens: int, completion_tokens: int, text: str) -> bytes:
|
||||
payload = {
|
||||
"candidates": [{"content": {"parts": [{"text": text}], "role": "model"}, "index": 0}],
|
||||
|
|
|
|||
|
|
@ -3454,10 +3454,11 @@ class TestResponsesInputTokens:
|
|||
body: dict[str, Any],
|
||||
path: str = "/v1/responses/input_tokens",
|
||||
counter: AsyncMock | None = None,
|
||||
llm_router: litellm.Router | None = None,
|
||||
) -> tuple[Response, AsyncMock]:
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _proxy_token_counter
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _proxy_llm_router, _proxy_token_counter
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
||||
token_counter_mock = (
|
||||
|
|
@ -3474,6 +3475,7 @@ class TestResponsesInputTokens:
|
|||
)
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test", request_route=path)
|
||||
app.dependency_overrides[_proxy_token_counter] = lambda: token_counter_mock
|
||||
app.dependency_overrides[_proxy_llm_router] = lambda: llm_router
|
||||
try:
|
||||
client = TestClient(app)
|
||||
response = client.post(path, json=body, headers={"Authorization": "Bearer sk-9876"})
|
||||
|
|
@ -3481,6 +3483,7 @@ class TestResponsesInputTokens:
|
|||
finally:
|
||||
app.dependency_overrides.pop(user_api_key_auth, None)
|
||||
app.dependency_overrides.pop(_proxy_token_counter, None)
|
||||
app.dependency_overrides.pop(_proxy_llm_router, None)
|
||||
|
||||
def test_string_input_returns_openai_input_tokens_shape(self):
|
||||
response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "Hello, how are you?"})
|
||||
|
|
@ -3594,6 +3597,48 @@ class TestResponsesInputTokens:
|
|||
assert response.status_code == 400, response.text
|
||||
error = response.json()["error"]
|
||||
assert error["type"] == "invalid_request_error"
|
||||
assert error["param"] == "tools"
|
||||
assert error["message"].startswith("tools: ")
|
||||
counter.assert_not_awaited()
|
||||
|
||||
def test_invalid_tools_are_dropped_under_drop_params(self) -> None:
|
||||
response, counter = self._post_input_tokens(
|
||||
{"model": "gpt-4o", "input": "hi", "tools": [5, {"type": "web_search"}], "drop_params": True}
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert counter.await_args.kwargs["request"].tools == [{"type": "web_search"}]
|
||||
|
||||
def test_malformed_tools_follow_the_counted_deployment_even_when_fallbacks_are_configured(self) -> None:
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-5.2",
|
||||
"litellm_params": {"model": "openai/gpt-5.2", "api_key": "sk-test", "additional_drop_params": ["tools"]},
|
||||
},
|
||||
{"model_name": "backup", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-test"}},
|
||||
],
|
||||
fallbacks=[{"gpt-5.2": ["backup"]}],
|
||||
)
|
||||
|
||||
response, counter = self._post_input_tokens(
|
||||
{"model": "gpt-5.2", "input": "hi", "tools": 5}, llm_router=router
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert counter.await_args.kwargs["request"].tools is None
|
||||
|
||||
def test_malformed_tools_for_a_model_outside_the_router_get_a_clean_400(self) -> None:
|
||||
router: Final = litellm.Router(
|
||||
model_list=[{"model_name": "gpt-5.2", "litellm_params": {"model": "openai/gpt-5.2", "api_key": "sk-test"}}]
|
||||
)
|
||||
|
||||
response, counter = self._post_input_tokens(
|
||||
{"model": "openai/gpt-5.4-mini", "input": "hi", "tools": 5}, llm_router=router
|
||||
)
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert response.json()["error"]["param"] == "tools"
|
||||
counter.assert_not_awaited()
|
||||
|
||||
def test_provider_error_maps_status_code(self):
|
||||
|
|
|
|||
|
|
@ -145,6 +145,21 @@ class TestExtractRequestToolNames:
|
|||
data = {"tools": [{"type": "function", "function": {"name": "x"}}]}
|
||||
assert extract_request_tool_names("/v1/embeddings", data) == []
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("route", "data"),
|
||||
[
|
||||
("/v1/chat/completions", {"tools": 5}),
|
||||
("/v1/chat/completions", {"functions": 5}),
|
||||
("/v1/responses", {"tools": 5}),
|
||||
("/v1/messages", {"tools": 5}),
|
||||
("/generate_content", {"tools": 5}),
|
||||
("/generate_content", {"tools": [{"functionDeclarations": 5}]}),
|
||||
],
|
||||
ids=["chat_tools", "chat_functions", "responses", "messages", "generate_content", "function_declarations"],
|
||||
)
|
||||
def test_malformed_tool_lists_yield_no_names(self, route: str, data: dict[str, object]) -> None:
|
||||
assert extract_request_tool_names(route, data) == []
|
||||
|
||||
|
||||
class TestCheckToolsAllowlist:
|
||||
"""Test allowlist enforcement in auth (no DB in hot path)."""
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ and ``_handle_logging_proxy_only_error``."""
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
|
@ -921,6 +922,48 @@ async def test_post_call_failure_hook_falls_back_to_body_shape_without_a_route(p
|
|||
assert request_data["call_type"] == "acompletion"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"call_type, body",
|
||||
[
|
||||
("anthropic_messages", {"model": "m", "messages": [{"role": "user", "content": "hi"}]}),
|
||||
("aresponses", {"model": "m", "input": [{"role": "user", "content": "hi"}]}),
|
||||
("acompact_responses", {"model": "m", "input": [{"role": "user", "content": "hi"}]}),
|
||||
],
|
||||
)
|
||||
async def test_post_call_failure_hook_keeps_the_call_type_request_setup_logged(
|
||||
proxy_logging: ProxyLogging,
|
||||
make_user_api_key_auth: Callable[..., UserAPIKeyAuth],
|
||||
call_type: str,
|
||||
body: dict[str, object],
|
||||
) -> None:
|
||||
"""A request rejected after request setup already carries its endpoint's call type, so the body
|
||||
shape must not relabel a /v1/messages rejection as a chat completion or a list-input /v1/responses
|
||||
rejection as an embedding."""
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
proxy_logging.alert_types = []
|
||||
request_data: Final = {
|
||||
**body,
|
||||
"litellm_logging_obj": LiteLLMLoggingObj(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type=call_type,
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id=f"test_keeps_{call_type}",
|
||||
function_id=f"test_keeps_{call_type}",
|
||||
),
|
||||
"metadata": {},
|
||||
}
|
||||
await proxy_logging.post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=HTTPException(status_code=400, detail="tools must be a list"),
|
||||
user_api_key_dict=make_user_api_key_auth(request_route="/v1/messages"),
|
||||
)
|
||||
assert request_data["call_type"] == call_type
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("route", ["/v1/files", "/files/file-abc", "/v1/containers"])
|
||||
async def test_post_call_failure_hook_keeps_the_route_for_multi_operation_routes(
|
||||
|
|
|
|||
|
|
@ -287,6 +287,19 @@ class TestFilterWebSearchDeployments:
|
|||
assert "deployment-2" in result_ids
|
||||
assert "deployment-3" not in result_ids
|
||||
|
||||
def test_web_search_in_a_tuple_filters_unsupported(self, sample_deployments: list[dict[str, object]]) -> None:
|
||||
result: Final = filter_web_search_deployments(sample_deployments, {"tools": ({"type": "web_search"},)})
|
||||
assert [d["model_info"]["id"] for d in result] == ["deployment-1", "deployment-2"]
|
||||
|
||||
@pytest.mark.parametrize("tool_type", [{}, [], None, 5], ids=["object", "list", "null", "int"])
|
||||
def test_a_non_string_tool_type_is_no_web_search_request(
|
||||
self, sample_deployments: list[dict[str, object]], tool_type: object
|
||||
) -> None:
|
||||
result: Final = filter_web_search_deployments(
|
||||
sample_deployments, {"tools": [{"type": tool_type}, {"type": "function"}]}
|
||||
)
|
||||
assert result == sample_deployments
|
||||
|
||||
def test_web_search_preview_filters_unsupported(self, sample_deployments):
|
||||
"""web_search_preview type should also trigger filtering"""
|
||||
request_kwargs = {"tools": [{"type": "web_search_preview"}]}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ the question neither covers: whether the job that globs a file then deselects it
|
|||
"""
|
||||
|
||||
import importlib.util
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
|
@ -71,17 +73,27 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path:
|
|||
workflow.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": "pytest tests/integration"}]}}}))
|
||||
_, findings = coverage._integration_ownership(tmp_path)
|
||||
assert [(finding.subject, finding.detail) for finding in findings] == [
|
||||
(test_path, "integration contract is also selected by GitHub Actions")
|
||||
(test_path, "integration file outside GITHUB_FILES is selected by GitHub Actions")
|
||||
]
|
||||
github_path: Final = "tests/integration/management/test_github_contract.py"
|
||||
github_path: Final = "tests/integration/database/image/test_github_contract.py"
|
||||
(tmp_path / github_path).parent.mkdir(parents=True)
|
||||
(tmp_path / github_path).write_text("def test_contract(): pass\n")
|
||||
runner: Final = tmp_path / "tests/integration/run.py"
|
||||
runner.write_text(runner.read_text() + f"GITHUB_FILES: Final = frozenset({{{github_path!r}}})\n")
|
||||
workflow.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}}))
|
||||
workflow.unlink()
|
||||
dedicated: Final = tmp_path / ".github/workflows/image-scan.yml"
|
||||
dedicated.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}}))
|
||||
github_owned, github_findings = coverage._integration_ownership(tmp_path)
|
||||
assert github_owned == frozenset({test_path, github_path})
|
||||
assert github_findings == ()
|
||||
workflow.write_text(yaml.safe_dump({"jobs": {}}))
|
||||
per_pr: Final = tmp_path / ".github/workflows/test-mcp-interactions.yml"
|
||||
per_pr.write_text(yaml.safe_dump({"jobs": {"mcp": {"steps": [{"run": f"pytest {github_path}"}]}}}))
|
||||
_, shared_findings = coverage._integration_ownership(tmp_path)
|
||||
assert [(finding.subject, finding.detail) for finding in shared_findings] == [
|
||||
(github_path, "GitHub-owned integration contract runs outside a dedicated integration workflow")
|
||||
]
|
||||
per_pr.unlink()
|
||||
dedicated.write_text(yaml.safe_dump({"jobs": {}}))
|
||||
_, missing_invocation = coverage._integration_ownership(tmp_path)
|
||||
assert [(finding.subject, finding.detail) for finding in missing_invocation] == [
|
||||
(github_path, "GitHub-owned integration contract has no invoking job")
|
||||
|
|
@ -94,6 +106,72 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path:
|
|||
assert circle_invocation == ()
|
||||
|
||||
|
||||
def _write_integration_tree(root: Path, github_path: str) -> None:
|
||||
contracts: Final = root / "tests/integration/management"
|
||||
contracts.mkdir(parents=True)
|
||||
(contracts / "test_contract.py").write_text("def test_contract(): pass\n")
|
||||
(root / github_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
(root / github_path).write_text("def test_contract(): pass\n")
|
||||
(root / "tests/integration/run.py").write_text(
|
||||
"from types import MappingProxyType\nfrom typing import Final\n"
|
||||
'GROUPS: Final = MappingProxyType({"management": ("management",)})\n'
|
||||
f"GITHUB_FILES: Final = frozenset({{{github_path!r}}})\n"
|
||||
)
|
||||
(root / ".circleci").mkdir()
|
||||
(root / ".circleci/config.yml").write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"jobs": {
|
||||
"integration_contracts": {
|
||||
"steps": [{"run": {"command": "bash .circleci/scripts/run_integration.sh management"}}]
|
||||
}
|
||||
},
|
||||
"workflows": {"integration": {"jobs": [{"integration_contracts": {"suite": "management"}}]}},
|
||||
}
|
||||
)
|
||||
)
|
||||
(root / ".github/workflows").mkdir(parents=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("github_path", "workflow_name", "exit_code"),
|
||||
[
|
||||
("tests/integration/database/image/test_github_contract.py", "image-scan.yml", 0),
|
||||
("tests/integration/database/image/test_github_contract.py", "test-unit.yml", 1),
|
||||
("tests/integration/management/test_github_contract.py", "image-scan.yml", 1),
|
||||
],
|
||||
)
|
||||
def test_census_fails_when_a_github_owned_contract_runs_outside_its_dedicated_workflow(
|
||||
tmp_path: Path, github_path: str, workflow_name: str, exit_code: int
|
||||
) -> None:
|
||||
script: Final = tmp_path / ".github/scripts/assert_ci_coverage.py"
|
||||
script.parent.mkdir(parents=True)
|
||||
shutil.copy(_MODULE_PATH, script)
|
||||
_write_integration_tree(tmp_path, github_path)
|
||||
(tmp_path / ".github/workflows" / workflow_name).write_text(
|
||||
yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}})
|
||||
)
|
||||
result: Final = subprocess.run([sys.executable, "-I", str(script)], capture_output=True, text=True, check=False)
|
||||
assert result.returncode == exit_code, result.stdout
|
||||
assert "Playwright" not in result.stdout
|
||||
|
||||
|
||||
def test_an_integration_file_outside_every_group_cannot_run_on_a_github_shard(tmp_path: Path) -> None:
|
||||
_write_integration_tree(tmp_path, "tests/integration/database/image/test_github_contract.py")
|
||||
ungrouped: Final = "tests/integration/ci_probe/test_ci_probe.py"
|
||||
(tmp_path / ungrouped).parent.mkdir(parents=True)
|
||||
(tmp_path / ungrouped).write_text("def test_probe(): pass\n")
|
||||
_, before = coverage._integration_ownership(tmp_path)
|
||||
(tmp_path / ".github/workflows/test-unit.yml").write_text(
|
||||
yaml.safe_dump({"jobs": {"mcp": {"steps": [{"run": f"pytest {ungrouped}"}]}}})
|
||||
)
|
||||
_, after = coverage._integration_ownership(tmp_path)
|
||||
assert [(finding.subject, finding.detail) for finding in after if finding.subject == ungrouped] == [
|
||||
(ungrouped, "integration file outside GITHUB_FILES is selected by GitHub Actions")
|
||||
]
|
||||
assert not [finding for finding in before if finding.subject == ungrouped]
|
||||
|
||||
|
||||
def test_circleci_postgres_suites_count_their_test_path_as_invoked() -> None:
|
||||
config: Final = {
|
||||
"workflows": {
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.otel.plumbing.context import set_request_root_span
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import closing_superseded_attempt
|
||||
from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import (
|
||||
SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES,
|
||||
)
|
||||
|
|
@ -9739,6 +9740,96 @@ def test_update_kwargs_with_deployment_merge_tools_request_overrides_tool_choice
|
|||
assert kwargs["tool_choice"] == "none"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_params, router_defaults, request_params, expected_tools",
|
||||
[
|
||||
({"drop_params": True}, {}, {}, [{"type": "web_search"}]),
|
||||
({}, {"drop_params": True}, {}, [{"type": "web_search"}]),
|
||||
({}, {}, {"additional_drop_params": ["tools"]}, [{"type": "web_search"}]),
|
||||
({"drop_params": True}, {}, {"drop_params": False}, 5),
|
||||
({}, {}, {}, 5),
|
||||
],
|
||||
ids=[
|
||||
"deployment_drop_params",
|
||||
"router_default_drop_params",
|
||||
"request_drops_tools",
|
||||
"request_overrides_deployment_drop",
|
||||
"no_drop_policy",
|
||||
],
|
||||
)
|
||||
def test_update_kwargs_with_deployment_keeps_deployment_tools_when_malformed_request_tools_drop(
|
||||
deployment_params: dict[str, object],
|
||||
router_defaults: dict[str, object],
|
||||
request_params: dict[str, object],
|
||||
expected_tools: object,
|
||||
) -> None:
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "o3-deep-research",
|
||||
"litellm_params": {
|
||||
"model": "openai/o3-deep-research",
|
||||
"api_key": "fake-key",
|
||||
"tools": [{"type": "web_search"}],
|
||||
**deployment_params,
|
||||
},
|
||||
},
|
||||
],
|
||||
default_litellm_params={**router_defaults},
|
||||
)
|
||||
kwargs: Final[dict[str, object]] = {"metadata": {}, "tools": 5, **request_params}
|
||||
router._update_kwargs_with_deployment(
|
||||
deployment=router.get_deployment_by_model_group_name(model_group_name="o3-deep-research"), kwargs=kwargs
|
||||
)
|
||||
|
||||
assert kwargs["tools"] == expected_tools
|
||||
|
||||
|
||||
_NAMED_REQUEST_TOOL_CHOICE: Final = {"type": "function", "function": {"name": "lookup_order"}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_params, expected_tool_fields",
|
||||
[
|
||||
(
|
||||
{"drop_params": True, "tool_choice": "required"},
|
||||
{"tools": [{"type": "web_search"}], "tool_choice": "required"},
|
||||
),
|
||||
({"drop_params": True}, {"tools": [{"type": "web_search"}]}),
|
||||
({}, {"tools": 5, "tool_choice": _NAMED_REQUEST_TOOL_CHOICE, "parallel_tool_calls": False}),
|
||||
],
|
||||
ids=["deployment_tool_choice_applies", "no_deployment_tool_choice", "no_drop_policy"],
|
||||
)
|
||||
def test_update_kwargs_with_deployment_drops_request_tool_settings_with_malformed_request_tools(
|
||||
deployment_params: dict[str, object], expected_tool_fields: dict[str, object]
|
||||
) -> None:
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "o3-deep-research",
|
||||
"litellm_params": {
|
||||
"model": "openai/o3-deep-research",
|
||||
"api_key": "fake-key",
|
||||
"tools": [{"type": "web_search"}],
|
||||
**deployment_params,
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
kwargs: Final[dict[str, object]] = {
|
||||
"metadata": {},
|
||||
"tools": 5,
|
||||
"tool_choice": _NAMED_REQUEST_TOOL_CHOICE,
|
||||
"parallel_tool_calls": False,
|
||||
}
|
||||
router._update_kwargs_with_deployment(
|
||||
deployment=router.get_deployment_by_model_group_name(model_group_name="o3-deep-research"), kwargs=kwargs
|
||||
)
|
||||
|
||||
tool_fields: Final = ("tools", "tool_choice", "parallel_tool_calls")
|
||||
assert {name: kwargs[name] for name in tool_fields if name in kwargs} == expected_tool_fields
|
||||
|
||||
|
||||
def test_credential_name_injected_as_tag():
|
||||
"""
|
||||
Test that litellm_credential_name from deployment litellm_params
|
||||
|
|
@ -16004,6 +16095,7 @@ class _AnthropicMessagesFakeByteStream:
|
|||
self._chunks = list(chunks)
|
||||
self._hidden_params = {"additional_headers": {"x-amzn-requestid": "req-1"}}
|
||||
self.closed = False
|
||||
self.closed_as_superseded = False
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
|
@ -16014,6 +16106,7 @@ class _AnthropicMessagesFakeByteStream:
|
|||
return self._chunks.pop(0)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
self.closed_as_superseded = self.closed_as_superseded or closing_superseded_attempt()
|
||||
self.closed = True
|
||||
|
||||
|
||||
|
|
@ -17256,6 +17349,30 @@ async def test_anthropic_messages_stream_dropped_before_content_is_retried_withi
|
|||
assert [(attempted, budget) for _, attempted, budget in provider.calls] == [(0, 2), (1, 2)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_superseded_stream_is_closed_before_the_retry_opens():
|
||||
"""Both attempts log through the request's one logging object, so the superseded attempt's end-of-stream
|
||||
logging must be queued before the retry starts producing its own: the router closes the dropped stream
|
||||
before it asks the group for another."""
|
||||
router = _anthropic_messages_retry_router(num_retries=1)
|
||||
superseded = _AnthropicMessagesFakeByteStream(
|
||||
[_anthropic_messages_message_start_chunk(), _anthropic_messages_overloaded_error_chunk()]
|
||||
)
|
||||
superseded_closed_when_retry_opened: list[bool] = []
|
||||
|
||||
def retry_stream():
|
||||
superseded_closed_when_retry_opened.append(superseded.closed_as_superseded)
|
||||
return _anthropic_messages_retried_stream()
|
||||
|
||||
provider = _AnthropicMessagesScriptedProvider(lambda: superseded, retry_stream)
|
||||
|
||||
stream = await _anthropic_messages_stream_through_router(router, provider)
|
||||
body = [chunk async for chunk in stream]
|
||||
|
||||
assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("pong")]
|
||||
assert superseded_closed_when_retry_opened == [True]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_ptu_share_holders_messages_stream_dropped_before_content_is_retried_on_its_shared_group(monkeypatch):
|
||||
"""The same-group retry has to look the group's health up as the calling team; looked up with no
|
||||
|
|
@ -18387,7 +18504,7 @@ async def test_aanthropic_messages_recover_stream_error_flushes_buffered_frames_
|
|||
|
||||
with patch.object(router, "_aanthropic_messages_fallback_attempt") as mock_attempt:
|
||||
recovery = router._aanthropic_messages_recover_stream_error(
|
||||
declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper()
|
||||
declined, True, buffered, "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
|
||||
)
|
||||
with pytest.raises(BedrockError) as exc_info:
|
||||
await drain(recovery)
|
||||
|
|
@ -18408,7 +18525,7 @@ async def test_aanthropic_messages_recover_stream_error_hands_converted_raise_to
|
|||
|
||||
with patch.object(router, "_aanthropic_messages_fallback_attempt", new=fake_attempt):
|
||||
recovery = router._aanthropic_messages_recover_stream_error(
|
||||
raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper()
|
||||
raised, False, (), "primary", {"model": "primary"}, _anthropic_messages_make_wrapper(), AsyncMock()
|
||||
)
|
||||
collected = [chunk async for chunk in recovery]
|
||||
assert collected == [b"fallback"]
|
||||
|
|
@ -18466,6 +18583,25 @@ async def test_anthropic_messages_post_first_chunk_error_skips_fallback():
|
|||
mock_fallback.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_stream_the_client_closes_at_a_forwarded_error_is_not_closed_as_superseded():
|
||||
"""An error frame forwarded to the client ends that request, and the SDK hangs up on it. No retry took the
|
||||
request over, so the source must not be closed as a superseded attempt, which would log it as a failure and
|
||||
drop the usage the provider billed for it."""
|
||||
router = _anthropic_messages_make_router()
|
||||
error_chunk = _anthropic_messages_overloaded_error_chunk()
|
||||
source = _AnthropicMessagesFakeByteStream([_anthropic_messages_content_chunk("partial answer"), error_chunk])
|
||||
|
||||
wrapped = await router._aanthropic_messages_streaming_iterator(response=source, initial_kwargs={"model": "primary"})
|
||||
async for chunk in wrapped:
|
||||
if chunk == error_chunk:
|
||||
break
|
||||
await wrapped.aclose()
|
||||
|
||||
assert source.closed is True
|
||||
assert source.closed_as_superseded is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_non_retriable_error_flushes_buffered_lifecycle_frames():
|
||||
"""A non-retriable error arriving while lifecycle frames are still
|
||||
|
|
|
|||
|
|
@ -349,6 +349,56 @@ async def test_malformed_vector_store_index_row_from_the_db_raises_a_validation_
|
|||
await VectorStoreIndexRegistry.get_vector_store_indexes_from_db(_database_listing_index_rows([row]))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tools", "expected"),
|
||||
[
|
||||
(5, []),
|
||||
(["vector_store_ids", {"type": "file_search", "vector_store_ids": ["vs_kept"]}], ["vs_kept"]),
|
||||
(
|
||||
[{"type": "file_search", "vector_store_ids": 5}, {"type": "file_search", "vector_store_ids": ["vs_kept"]}],
|
||||
["vs_kept"],
|
||||
),
|
||||
],
|
||||
ids=["int", "string_entry", "int_vector_store_ids"],
|
||||
)
|
||||
def test_vector_store_ids_to_run_skips_malformed_tools(tools: object, expected: list[str]) -> None:
|
||||
assert VectorStoreRegistry().get_vector_store_ids_to_run(non_default_params={}, tools=tools) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_store_ids",
|
||||
["vs_requested", ["vs_requested", 5]],
|
||||
ids=["string", "list_with_a_non_string"],
|
||||
)
|
||||
def test_vector_store_ids_to_run_reads_every_store_id_a_malformed_vector_store_ids_field_names(
|
||||
vector_store_ids: object,
|
||||
) -> None:
|
||||
tools: Final = [{"type": "file_search", "vector_store_ids": vector_store_ids}]
|
||||
assert VectorStoreRegistry().get_vector_store_ids_to_run(non_default_params={}, tools=tools) == ["vs_requested"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_store_ids",
|
||||
["vs_allowed", ["vs_allowed", 12345], [12345], ["vs_allowed"]],
|
||||
ids=["string", "list_with_a_non_string", "only_non_strings", "list_of_strings"],
|
||||
)
|
||||
def test_vector_store_tools_pop_exactly_the_store_ids_the_registry_reports_to_run(vector_store_ids: object) -> None:
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
registry: Final = VectorStoreRegistry(
|
||||
[
|
||||
LiteLLM_ManagedVectorStore(vector_store_id=vector_store_id, created_at=now, updated_at=now)
|
||||
for vector_store_id in ("vs_allowed", 12345)
|
||||
]
|
||||
)
|
||||
tools: Final = [{"type": "file_search", "vector_store_ids": vector_store_ids}]
|
||||
checked: Final = registry.get_vector_store_ids_to_run(non_default_params={}, tools=tools)
|
||||
ran: Final[list[str]] = []
|
||||
|
||||
registry.get_and_pop_recognised_vector_store_tools(tools=[dict(tool) for tool in tools], vector_store_ids=ran)
|
||||
|
||||
assert ran == checked
|
||||
|
||||
|
||||
class TestRegistriesDoNotShareDefaultState:
|
||||
def test_adding_to_one_registry_leaves_the_next_one_empty(self):
|
||||
VectorStoreRegistry().add_vector_store_to_registry(
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@ import { Button } from "@/components/ui/button";
|
|||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { PasswordInput } from "@/components/shared/PasswordInput";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { MultiSelect } from "@/components/shared/MultiSelect";
|
||||
import DcrBridgeToggle from "./DcrBridgeToggle";
|
||||
import { MountedFormField } from "@/components/common_components/MountedFormField";
|
||||
import { textControl } from "./mcpFieldRules";
|
||||
import { tagsControl, textControl } from "./mcpFieldRules";
|
||||
import { credentialAuthClass, isClientForwardedTokenMode } from "@/components/mcp_tools/types";
|
||||
|
||||
interface PassthroughOAuthFlow {
|
||||
|
|
@ -69,6 +71,7 @@ export default function PassthroughAuthorizeSection({
|
|||
const clientSecretPlaceholder = classUnchanged
|
||||
? "Leave blank to keep the currently saved secret (if any)"
|
||||
: "Leave blank for public clients / PKCE";
|
||||
const fieldClassName = "rounded-lg border-border focus:border-info focus:ring-ring";
|
||||
const clientIdExtra = classUnchanged
|
||||
? "Set this to make everyone authorize through a specific app; required for upstreams without dynamic client registration (e.g. a pre-registered Slack app)."
|
||||
: "Switching the auth type discards the previously saved app; enter a client ID here or leave blank to use dynamic client registration.";
|
||||
|
|
@ -113,6 +116,39 @@ export default function PassthroughAuthorizeSection({
|
|||
/>
|
||||
)}
|
||||
</MountedFormField>
|
||||
<MountedFormField
|
||||
label={<span className="text-sm font-medium text-foreground">Authorization URL (optional)</span>}
|
||||
name="authorization_url"
|
||||
help="Set this when the upstream publishes no OAuth metadata, for example an internal server behind Microsoft Entra. Leave blank to discover it from the upstream."
|
||||
>
|
||||
{(control) => (
|
||||
<Input
|
||||
{...textControl(control)}
|
||||
placeholder="https://login.microsoftonline.com/<tenant-id>/oauth2/v2.0/authorize"
|
||||
className={fieldClassName}
|
||||
/>
|
||||
)}
|
||||
</MountedFormField>
|
||||
<MountedFormField
|
||||
label={<span className="text-sm font-medium text-foreground">Token URL (optional)</span>}
|
||||
name="token_url"
|
||||
help="Token endpoint of the same authorization server. Leave blank to discover it from the upstream."
|
||||
>
|
||||
{(control) => (
|
||||
<Input
|
||||
{...textControl(control)}
|
||||
placeholder="https://login.microsoftonline.com/<tenant-id>/oauth2/v2.0/token"
|
||||
className={fieldClassName}
|
||||
/>
|
||||
)}
|
||||
</MountedFormField>
|
||||
<MountedFormField
|
||||
label={<span className="text-sm font-medium text-foreground">Scopes (optional)</span>}
|
||||
name={["credentials", "scopes"]}
|
||||
help="Scopes requested from the upstream when a user authorizes. Press enter or comma between scopes."
|
||||
>
|
||||
{(control) => <MultiSelect {...tagsControl(control)} placeholder="Add scopes" className="rounded-lg" />}
|
||||
</MountedFormField>
|
||||
<DcrBridgeToggle authType={authType} initialChecked={dcrBridgeInitialChecked} />
|
||||
{isEditing && onRemoveStoredAppChange && (
|
||||
<Label className="items-start leading-normal font-normal text-foreground">
|
||||
|
|
|
|||
|
|
@ -201,6 +201,22 @@ describe("buildCreateServerPayload credentials", () => {
|
|||
},
|
||||
);
|
||||
|
||||
it.each([["true_passthrough"], ["oauth_delegate"]])(
|
||||
"carries the admin-entered authorization URL, token URL and scopes for %s",
|
||||
(authType) => {
|
||||
const entraServer = {
|
||||
auth_type: authType,
|
||||
authorization_url: "https://login.microsoftonline.com/tenant/oauth2/v2.0/authorize",
|
||||
token_url: "https://login.microsoftonline.com/tenant/oauth2/v2.0/token",
|
||||
credentials: { client_id: "org-app", scopes: ["api://mcp/.default", "offline_access", ""] },
|
||||
};
|
||||
const payload = payloadOf(build(entraServer));
|
||||
expect(payload.authorization_url).toBe("https://login.microsoftonline.com/tenant/oauth2/v2.0/authorize");
|
||||
expect(payload.token_url).toBe("https://login.microsoftonline.com/tenant/oauth2/v2.0/token");
|
||||
expect(payload.credentials).toEqual({ client_id: "org-app", scopes: ["api://mcp/.default", "offline_access"] });
|
||||
},
|
||||
);
|
||||
|
||||
it("merges the DCR-minted client into an oauth2 payload", () => {
|
||||
const payload = payloadOf(
|
||||
build(
|
||||
|
|
|
|||
|
|
@ -295,11 +295,6 @@ export const CASES: readonly DifferentialCase[] = [
|
|||
ui: {},
|
||||
},
|
||||
{ label: "credentials absent entirely", values: { ...ROOT, auth_type: "oauth2" }, ui: {} },
|
||||
{
|
||||
label: "removeStoredApp forces an explicit-null app write",
|
||||
values: { ...ROOT, auth_type: "true_passthrough", credentials: { ...CREDS } },
|
||||
ui: { removeStoredApp: true, mcpServer: { ...SERVER, auth_type: "true_passthrough" } },
|
||||
},
|
||||
{
|
||||
label: "removeStoredApp ignored outside a client-forwarded mode",
|
||||
values: { ...ROOT, auth_type: "api_key", credentials: { ...CREDS } },
|
||||
|
|
|
|||
|
|
@ -398,3 +398,61 @@ it.each([
|
|||
expect(result.payload.credentials?.client_secret).toBe(useDcr ? null : "configured-secret");
|
||||
expect(result.payload.credentials?.dcr_issuer).toBe(useDcr ? "https://new.example" : undefined);
|
||||
});
|
||||
|
||||
it.each([AUTH_TYPE.OAUTH_DELEGATE, AUTH_TYPE.TRUE_PASSTHROUGH])(
|
||||
"keeps the endpoints typed for %s when the server was saved as oauth2",
|
||||
(authType) => {
|
||||
const authorizationUrl = "https://login.microsoftonline.com/tenant/oauth2/v2.0/authorize";
|
||||
const tokenUrl = "https://login.microsoftonline.com/tenant/oauth2/v2.0/token";
|
||||
const values: EditServerFormValues = {
|
||||
auth_type: authType,
|
||||
transport: "http",
|
||||
authorization_url: authorizationUrl,
|
||||
token_url: tokenUrl,
|
||||
};
|
||||
const ui: EditServerUiState = { ...baseUi, mcpServer: { ...baseUi.mcpServer, auth_type: AUTH_TYPE.OAUTH2 } };
|
||||
const result = buildEditServerPayload(values, ui);
|
||||
expect(result.kind).toBe("ok");
|
||||
if (result.kind !== "ok") throw new Error("Expected valid MCP server payload");
|
||||
expect(result.payload.authorization_url).toBe(authorizationUrl);
|
||||
expect(result.payload.token_url).toBe(tokenUrl);
|
||||
expect(result.payload.issuer).toBeNull();
|
||||
expect(result.payload.registration_url).toBeNull();
|
||||
},
|
||||
);
|
||||
|
||||
it.each([AUTH_TYPE.OAUTH_DELEGATE, AUTH_TYPE.TRUE_PASSTHROUGH])(
|
||||
"writes an explicit null when the admin removes every scope on a %s server",
|
||||
(authType) => {
|
||||
const values: EditServerFormValues = {
|
||||
auth_type: authType,
|
||||
transport: "http",
|
||||
credentials: { client_id: "org-app", scopes: [""] },
|
||||
};
|
||||
const ui: EditServerUiState = { ...baseUi, mcpServer: { ...baseUi.mcpServer, auth_type: authType } };
|
||||
const result = buildEditServerPayload(values, ui);
|
||||
expect(result.kind).toBe("ok");
|
||||
if (result.kind !== "ok") throw new Error("Expected valid MCP server payload");
|
||||
expect(result.payload.credentials).toEqual({ client_id: "org-app", scopes: null });
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
[["read"], ["read"]],
|
||||
[[""], null],
|
||||
])("keeps the scope edit %j when the saved app is removed on a client-forwarded server", (typed, expected) => {
|
||||
const values: EditServerFormValues = {
|
||||
auth_type: AUTH_TYPE.TRUE_PASSTHROUGH,
|
||||
transport: "http",
|
||||
credentials: { client_id: "cid", client_secret: "csec", scopes: typed },
|
||||
};
|
||||
const ui: EditServerUiState = {
|
||||
...baseUi,
|
||||
removeStoredApp: true,
|
||||
mcpServer: { ...baseUi.mcpServer, auth_type: AUTH_TYPE.TRUE_PASSTHROUGH },
|
||||
};
|
||||
const result = buildEditServerPayload(values, ui);
|
||||
expect(result.kind).toBe("ok");
|
||||
if (result.kind !== "ok") throw new Error("Expected valid MCP server payload");
|
||||
expect(result.payload.credentials).toEqual({ scopes: expected, client_id: null, client_secret: null });
|
||||
});
|
||||
|
|
|
|||
|
|
@ -214,6 +214,21 @@ const buildCredentials = (credentialValues: unknown): Readonly<Record<string, un
|
|||
return Object.fromEntries(kept);
|
||||
};
|
||||
|
||||
const unmountedOauth2Endpoints = (authType: string | undefined) =>
|
||||
isClientForwardedTokenMode(authType) ? {} : { authorization_url: null, token_url: null };
|
||||
|
||||
const scopesCleared = (credentialValues: unknown): boolean => {
|
||||
if (!credentialValues || typeof credentialValues !== "object") return false;
|
||||
const scopes: unknown = (credentialValues as { scopes?: unknown }).scopes;
|
||||
return Array.isArray(scopes) && scopes.every((scope: unknown) => scope == null || scope === "");
|
||||
};
|
||||
|
||||
const withClearedScopes = (
|
||||
credentials: Readonly<Record<string, unknown>> | undefined,
|
||||
credentialValues: unknown,
|
||||
): Readonly<Record<string, unknown>> | undefined =>
|
||||
scopesCleared(credentialValues) ? { ...(credentials ?? {}), scopes: null } : credentials;
|
||||
|
||||
interface CredentialsEntryInput {
|
||||
readonly dcrClient?: McpDcrCredentials | null;
|
||||
readonly oauthFlow?: string;
|
||||
|
|
@ -239,7 +254,7 @@ const resolveCredentialsEntry = ({
|
|||
return { credentials: { ...credentials, ...dcrClient } };
|
||||
}
|
||||
if (removeStoredApp && isClientForwardedTokenMode(authType)) {
|
||||
return { credentials: { client_id: null, client_secret: null } };
|
||||
return { credentials: { ...(credentials ?? {}), client_id: null, client_secret: null } };
|
||||
}
|
||||
if (includeCredentials && credentials && Object.keys(credentials).length > 0) {
|
||||
return { credentials };
|
||||
|
|
@ -334,7 +349,7 @@ export const buildEditServerPayload = (values: EditServerFormValues, ui: EditSer
|
|||
// Client-forwarded rows persist ONLY the declared app; strip any token material lingering in the
|
||||
// form (e.g. from a prior oauth2 authorize this session) so it can never reach the row.
|
||||
const submitCredentials = isClientForwardedTokenMode(restValues.auth_type)
|
||||
? preservedAdminCredentials(credentialsPayload)
|
||||
? withClearedScopes(preservedAdminCredentials(credentialsPayload), credentialValues)
|
||||
: credentialsPayload;
|
||||
const includeCredentials = restValues.auth_type && AUTH_TYPES_REQUIRING_CREDENTIALS.includes(restValues.auth_type);
|
||||
|
||||
|
|
@ -355,7 +370,7 @@ export const buildEditServerPayload = (values: EditServerFormValues, ui: EditSer
|
|||
stdio_config: undefined,
|
||||
env_json: undefined,
|
||||
...(mcpServer.auth_type === AUTH_TYPE.OAUTH2 && restValues.auth_type !== AUTH_TYPE.OAUTH2
|
||||
? { issuer: null, authorization_url: null, token_url: null, registration_url: null }
|
||||
? { issuer: null, ...unmountedOauth2Endpoints(restValues.auth_type), registration_url: null }
|
||||
: {}),
|
||||
...(mcpServer.auth_type === AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE &&
|
||||
restValues.auth_type !== AUTH_TYPE.OAUTH2_TOKEN_EXCHANGE
|
||||
|
|
|
|||
|
|
@ -264,6 +264,8 @@ const CASES: readonly Case[] = [
|
|||
...EXPECTED_BASE,
|
||||
auth_type: "true_passthrough",
|
||||
credentials: { client_id: "cid", client_secret: "csec" },
|
||||
authorization_url: undefined,
|
||||
token_url: undefined,
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -273,6 +275,8 @@ const CASES: readonly Case[] = [
|
|||
...EXPECTED_BASE,
|
||||
auth_type: "oauth_delegate",
|
||||
credentials: { client_id: "cid", client_secret: "csec" },
|
||||
authorization_url: undefined,
|
||||
token_url: undefined,
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -283,7 +287,13 @@ const CASES: readonly Case[] = [
|
|||
{
|
||||
label: "dcr_bridge stays true for a client-forwarded mode",
|
||||
server: { ...BASE, auth_type: "true_passthrough", dcr_bridge: true },
|
||||
expected: { ...EXPECTED_BASE, auth_type: "true_passthrough", dcr_bridge: true },
|
||||
expected: {
|
||||
...EXPECTED_BASE,
|
||||
auth_type: "true_passthrough",
|
||||
dcr_bridge: true,
|
||||
authorization_url: undefined,
|
||||
token_url: undefined,
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "an existing allowed_tools list enforces the allowlist and round-trips",
|
||||
|
|
|
|||
|
|
@ -333,6 +333,14 @@ describe("preservedAdminCredentials vs preservedDeclaredAppCredentials", () => {
|
|||
expect(preservedAdminCredentials(credentials)).toEqual({ client_id: "cid", upstream_resource: "auto" });
|
||||
});
|
||||
|
||||
it("preserves admin-typed scopes and drops blank ones", () => {
|
||||
expect(preservedAdminCredentials({ client_id: "cid", scopes: ["api://mcp/.default", ""] })).toEqual({
|
||||
client_id: "cid",
|
||||
scopes: ["api://mcp/.default"],
|
||||
});
|
||||
expect(preservedAdminCredentials({ client_id: "cid", scopes: [] })).toEqual({ client_id: "cid" });
|
||||
});
|
||||
|
||||
// The two helpers answer different questions and must not be collapsed: "has the admin declared an
|
||||
// OAuth app" gates the app-may-not-match-upstream warning, so a resource-only server must read as
|
||||
// having no declared app.
|
||||
|
|
|
|||
|
|
@ -181,10 +181,23 @@ export const preservedDeclaredAppCredentials = (
|
|||
// Everything the admin typed into `credentials` and nothing minted: the declared app plus the config
|
||||
// keys. This is what must survive the invalidation reset and what a client-forwarded row may persist,
|
||||
// so dropping a key from here silently discards admin input on an unrelated edit.
|
||||
const adminScopes = (value: unknown): string[] | undefined => {
|
||||
if (!Array.isArray(value)) return undefined;
|
||||
const kept = value.filter((scope): scope is string => typeof scope === "string" && scope !== "");
|
||||
return kept.length > 0 ? kept : undefined;
|
||||
};
|
||||
|
||||
export const preservedAdminCredentials = (
|
||||
credentials: Record<string, unknown> | null | undefined,
|
||||
): Record<string, string> | undefined =>
|
||||
pickStringCredentials(credentials, [...DECLARED_APP_CREDENTIAL_KEYS, ...ADMIN_CONFIG_CREDENTIAL_KEYS]);
|
||||
): Record<string, string | string[]> | undefined => {
|
||||
const strings = pickStringCredentials(credentials, [
|
||||
...DECLARED_APP_CREDENTIAL_KEYS,
|
||||
...ADMIN_CONFIG_CREDENTIAL_KEYS,
|
||||
]);
|
||||
const scopes = adminScopes(credentials?.scopes);
|
||||
if (scopes === undefined) return strings;
|
||||
return { ...(strings ?? {}), scopes };
|
||||
};
|
||||
|
||||
// Drop minted token keys, keeping everything else (the declared app plus any non-token config).
|
||||
export const withoutMintedTokenCredentials = (
|
||||
|
|
|
|||
12
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
12
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -26991,6 +26991,12 @@ export interface components {
|
|||
* @default true
|
||||
*/
|
||||
fail_on_error: boolean | null;
|
||||
/**
|
||||
* Forward Messages As Sent
|
||||
* @description Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can carry provider-native content blocks and keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with undeclared keys removed.
|
||||
* @default false
|
||||
*/
|
||||
forward_messages_as_sent: boolean | null;
|
||||
/**
|
||||
* Guard Name
|
||||
* @description Name of the guardrail in guardrails.ai
|
||||
|
|
@ -36471,6 +36477,12 @@ export interface components {
|
|||
* @description HTTP timeout in seconds for checking attached files. Default: 10.
|
||||
*/
|
||||
file_guardrail_timeout?: number | null;
|
||||
/**
|
||||
* Forward Messages As Sent
|
||||
* @description Only honored by guardrail='generic_guardrail_api'. When True, structured_messages reaches the guardrail exactly as the client sent it, so rows can carry provider-native content blocks and keys the OpenAI chat types do not declare. Default False sends OpenAI-format chat rows with undeclared keys removed.
|
||||
* @default false
|
||||
*/
|
||||
forward_messages_as_sent: boolean | null;
|
||||
/**
|
||||
* Gateway Name
|
||||
* @description noma_v2 only: name of this gateway, used as the gateway_host label on Noma scans
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue