From 92edcb90dbb41175f526187826f70058e5a57f94 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:27:19 -0700 Subject: [PATCH 1/3] fix: keep litellm importable on Python 3.10 and guard 3.11-only typing imports in CI (#39448) * ci: guard against Python 3.10-incompatible typing imports Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: address Python 3.10 typing guard review Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ci): honor version-guard direction and scan litellm-proxy-extras in py310 typing check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-code-quality.yml | 33 ++++ .../websearch_interception/handler.py | 4 +- .../_experimental/mcp_server/tool_search.py | 4 +- litellm/proxy/agent_endpoints/endpoints.py | 4 +- .../proxy/common_utils/reset_budget_job.py | 4 +- .../batch_file_validation.py | 4 +- .../general_upload_validation.py | 4 +- .../types/llms/gemini_audio_transcription.py | 4 +- .../check_py310_typing_imports.py | 150 ++++++++++++++++++ .../test_check_py310_typing_imports.py | 86 ++++++++++ 10 files changed, 286 insertions(+), 11 deletions(-) create mode 100644 tests/code_coverage_tests/check_py310_typing_imports.py create mode 100644 tests/test_litellm/test_check_py310_typing_imports.py diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index c112bf2bb22..d02f5878396 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -128,6 +128,9 @@ jobs: - name: check_fastuuid_usage run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py + - name: check_py310_typing_imports + run: uv run --no-sync python ./tests/code_coverage_tests/check_py310_typing_imports.py + - name: check_e2e_no_raw_requests run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py @@ -145,3 +148,33 @@ jobs: - name: documentation_test_api_docs run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py + + python-310-import-smoke: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.10" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Install dependencies + run: uv sync --frozen --extra proxy --python 3.10 + + - run: uv run --no-sync python --version + + - name: Import litellm + run: uv run --no-sync python -c "import litellm" + + - name: Check litellm CLI + run: uv run --no-sync litellm --version diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 2d737bc34e7..587da997f94 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -10,9 +10,9 @@ import asyncio import math import uuid from collections.abc import AsyncIterator, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Literal, Never, TypedDict, TypeVar, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast -from typing_extensions import ReadOnly +from typing_extensions import Never, ReadOnly import litellm from litellm._logging import verbose_logger diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/_experimental/mcp_server/tool_search.py index 4f6305d88cf..af02c11ad86 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_search.py +++ b/litellm/proxy/_experimental/mcp_server/tool_search.py @@ -5,10 +5,10 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, TypedDict, assert_never +from typing import TYPE_CHECKING, Any, Final, TypedDict from pydantic import ValidationError -from typing_extensions import ReadOnly, Required +from typing_extensions import ReadOnly, Required, assert_never import litellm from litellm.proxy.agent_endpoints.agent_search import DEFAULT_AGENT_SEARCH_TOP_K diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index 3e4dc07a521..3b8151d1064 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -13,10 +13,10 @@ import os import uuid from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Annotated, Final, TypedDict, assert_never +from typing import Annotated, Final, TypedDict from fastapi import APIRouter, Depends, HTTPException, Query, Request -from typing_extensions import ReadOnly, Required +from typing_extensions import ReadOnly, Required, assert_never import litellm from litellm._logging import verbose_proxy_logger diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 1682cf12f4e..47f69732e95 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -7,7 +7,9 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from enum import Enum from types import MappingProxyType -from typing import Final, Literal, Protocol, TypeVar, assert_never +from typing import Final, Literal, Protocol, TypeVar + +from typing_extensions import assert_never import litellm from litellm._logging import verbose_proxy_logger diff --git a/litellm/proxy/openai_files_endpoints/batch_file_validation.py b/litellm/proxy/openai_files_endpoints/batch_file_validation.py index 0aee5e8cc54..a41bd36d510 100644 --- a/litellm/proxy/openai_files_endpoints/batch_file_validation.py +++ b/litellm/proxy/openai_files_endpoints/batch_file_validation.py @@ -2,7 +2,9 @@ import json from collections.abc import Iterator from dataclasses import dataclass from itertools import chain -from typing import BinaryIO, Final, NoReturn, assert_never +from typing import BinaryIO, Final, NoReturn + +from typing_extensions import assert_never from litellm.proxy._types import ProxyException diff --git a/litellm/proxy/openai_files_endpoints/general_upload_validation.py b/litellm/proxy/openai_files_endpoints/general_upload_validation.py index 9d450cb5b8d..8c59a520272 100644 --- a/litellm/proxy/openai_files_endpoints/general_upload_validation.py +++ b/litellm/proxy/openai_files_endpoints/general_upload_validation.py @@ -8,7 +8,9 @@ extensions, path-traversal filenames) regardless of purpose. from dataclasses import dataclass from pathlib import Path -from typing import BinaryIO, Final, NoReturn, assert_never +from typing import BinaryIO, Final, NoReturn + +from typing_extensions import assert_never from litellm.proxy._types import ProxyException from litellm.proxy.common_utils.path_utils import safe_filename diff --git a/litellm/types/llms/gemini_audio_transcription.py b/litellm/types/llms/gemini_audio_transcription.py index cb12e0f45b8..f7e74ba4bf8 100644 --- a/litellm/types/llms/gemini_audio_transcription.py +++ b/litellm/types/llms/gemini_audio_transcription.py @@ -1,7 +1,7 @@ -from typing import Literal, Required +from typing import Literal from pydantic import BaseModel, ConfigDict -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, Required, TypedDict class GeminiTranscriptionAudioInput(TypedDict): diff --git a/tests/code_coverage_tests/check_py310_typing_imports.py b/tests/code_coverage_tests/check_py310_typing_imports.py new file mode 100644 index 00000000000..0cd4d089890 --- /dev/null +++ b/tests/code_coverage_tests/check_py310_typing_imports.py @@ -0,0 +1,150 @@ +import ast +import os +import sys +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +PY311_PLUS_TYPING_NAMES: Final[frozenset[str]] = frozenset( + { + "NotRequired", + "Required", + "Self", + "LiteralString", + "Never", + "assert_never", + "assert_type", + "reveal_type", + "TypeVarTuple", + "Unpack", + "dataclass_transform", + "override", + "TypeAliasType", + "get_original_bases", + "ReadOnly", + "TypeIs", + "NoDefault", + "get_protocol_members", + "is_protocol", + "evaluate_forward_ref", + "TypeForm", + } +) + + +@dataclass(frozen=True, slots=True) +class TypingImportViolation: + file: str + line: int + name: str + + +def _walk_with_ancestors( + node: ast.AST, ancestors: tuple[tuple[ast.AST, str], ...] = () +) -> Iterator[tuple[ast.AST, tuple[tuple[ast.AST, str], ...]]]: + yield node, ancestors + for field_name, field_value in ast.iter_fields(node): + if isinstance(field_value, ast.AST): + yield from _walk_with_ancestors(field_value, (*ancestors, (node, field_name))) + elif isinstance(field_value, list): + for child in field_value: + if isinstance(child, ast.AST): + yield from _walk_with_ancestors(child, (*ancestors, (node, field_name))) + + +def _is_sys_version_info(node: ast.AST) -> bool: + return ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "sys" + and node.attr == "version_info" + ) + + +def _is_version_guarded(ancestors: tuple[tuple[ast.AST, str], ...]) -> bool: + nearest_if: Final[tuple[ast.If, str] | None] = next( + ( + (ancestor, field_name) + for ancestor, field_name in reversed(ancestors) + if isinstance(ancestor, ast.If) + ), + None, + ) + if nearest_if is None: + return False + enclosing_if, branch = nearest_if + test: Final[ast.expr] = enclosing_if.test + if not isinstance(test, ast.Compare) or len(test.ops) != 1 or not _is_sys_version_info(test.left): + return False + operator: Final[ast.cmpop] = test.ops[0] + return (isinstance(operator, (ast.Gt, ast.GtE)) and branch == "body") or ( + isinstance(operator, (ast.Lt, ast.LtE)) and branch == "orelse" + ) + + +def scan_file(file_path: str | os.PathLike[str]) -> tuple[TypingImportViolation, ...]: + path: Final[Path] = Path(file_path) + tree: Final[ast.Module] = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + return tuple( + violation + for node, ancestors in _walk_with_ancestors(tree) + if not _is_version_guarded(ancestors) + for violation in _violations_for_node(node, path) + ) + + +def _violations_for_node( + node: ast.AST, path: Path +) -> tuple[TypingImportViolation, ...]: + if isinstance(node, ast.ImportFrom) and node.module == "typing": + return tuple( + TypingImportViolation(file=str(path), line=node.lineno, name=alias.name) + for alias in node.names + if alias.name in PY311_PLUS_TYPING_NAMES + ) + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "typing" + and node.attr in PY311_PLUS_TYPING_NAMES + ): + return (TypingImportViolation(file=str(path), line=node.lineno, name=node.attr),) + return () + + +def scan_directory(base_dir: str | os.PathLike[str] = ".") -> tuple[TypingImportViolation, ...]: + base_path: Final[Path] = Path(base_dir) + return tuple( + violation + for directory in ( + base_path / "litellm", + base_path / "enterprise", + base_path / "litellm-proxy-extras" / "litellm_proxy_extras", + ) + if directory.exists() + for path in directory.rglob("*.py") + for violation in scan_file(path) + ) + + +def main() -> None: + violations: Final[tuple[TypingImportViolation, ...]] = scan_directory() + if violations: + message: Final[str] = "\n".join( + ( + "Python 3.10-incompatible typing imports found:", + *( + f"{violation.file}:{violation.line}: {violation.name} is unavailable in Python 3.10; " + "import it from typing_extensions instead because litellm supports Python 3.10" + for violation in violations + ), + ) + ) + sys.stdout.write(f"{message}\n") + raise RuntimeError("Import Python 3.10-incompatible typing names from typing_extensions instead") + sys.stdout.write("No Python 3.10-incompatible typing imports found.\n") + + +if __name__ == "__main__": + main() diff --git a/tests/test_litellm/test_check_py310_typing_imports.py b/tests/test_litellm/test_check_py310_typing_imports.py new file mode 100644 index 00000000000..de370326091 --- /dev/null +++ b/tests/test_litellm/test_check_py310_typing_imports.py @@ -0,0 +1,86 @@ +import sys +from pathlib import Path +from typing import Final + +_CODE_COVERAGE_DIR: Final[Path] = Path(__file__).resolve().parents[1] / "code_coverage_tests" +sys.path.insert(0, str(_CODE_COVERAGE_DIR)) # test-quality-ok: required to import checker from its source directory +import check_py310_typing_imports as checker # noqa: E402 # load checker from its source directory + + +def _scan(tmp_path: Path, source: str) -> tuple[object, ...]: + file_path = tmp_path / "fixture.py" + file_path.write_text(source, encoding="utf-8") + return checker.scan_file(file_path) + + +def test_typing_import_flags_python_311_name(tmp_path: Path) -> None: + violations = _scan(tmp_path, "from typing import NotRequired, TypedDict\n") + assert tuple(violation.name for violation in violations) == ("NotRequired",) + + +def test_typing_extensions_import_passes(tmp_path: Path) -> None: + assert _scan(tmp_path, "from typing_extensions import NotRequired\n") == () + + +def test_typing_attribute_flags_python_311_name(tmp_path: Path) -> None: + violations = _scan(tmp_path, "import typing\nx: typing.Self\n") + assert tuple(violation.name for violation in violations) == ("Self",) + + +def test_version_guarded_typing_import_passes(tmp_path: Path) -> None: + source = ( + "import sys\n" + "if sys.version_info >= (3, 11):\n" + " from typing import NotRequired\n" + "else:\n" + " from typing_extensions import NotRequired\n" + ) + assert _scan(tmp_path, source) == () + + +def test_python_310_branch_flags_typing_import(tmp_path: Path) -> None: + source = ( + "import sys\n" + "if sys.version_info >= (3, 11):\n" + " from typing_extensions import NotRequired\n" + "else:\n" + " from typing import NotRequired\n" + ) + violations = _scan(tmp_path, source) + assert tuple(violation.name for violation in violations) == ("NotRequired",) + + +def test_python_310_branch_is_exempt_for_less_than_guard(tmp_path: Path) -> None: + source = ( + "import sys\n" + "if sys.version_info < (3, 11):\n" + " from typing_extensions import NotRequired\n" + "else:\n" + " from typing import NotRequired\n" + ) + assert _scan(tmp_path, source) == () + + +def test_nearest_if_controls_version_guard(tmp_path: Path) -> None: + source = ( + "if sys.version_info >= (3, 11):\n" + " from typing import Self\n" + " x = 1\n" + "if True:\n" + " from typing import Self\n" + ) + violations = _scan(tmp_path, source) + assert tuple((violation.name, violation.line) for violation in violations) == (("Self", 5),) + + +def test_scan_directory_includes_proxy_extras(tmp_path: Path) -> None: + file_path = tmp_path / "litellm-proxy-extras" / "litellm_proxy_extras" / "m.py" + file_path.parent.mkdir(parents=True) + file_path.write_text("from typing import NotRequired\n", encoding="utf-8") + + violations = checker.scan_directory(tmp_path) + assert tuple((violation.name, violation.file) for violation in violations) == (("NotRequired", str(file_path)),) + + +def test_python_310_typing_name_passes(tmp_path: Path) -> None: + assert _scan(tmp_path, "from typing import Optional\n") == () From 8441dd6e8ca16a480e0766651f7c4ab99d22d5d5 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:28:06 -0700 Subject: [PATCH 2/3] fix(proxy): keep SpendLogs and callback session ids in sync when the request has none (#39450) * fix(proxy): keep SpendLogs and callback session ids in sync when the request has none Add general_settings.missing_session_id (generate | reject). In generate mode one id is stamped into litellm_session_id, litellm_trace_id and metadata.session_id before callbacks run, so LiteLLM_SpendLogs.session_id and the Langfuse session id match. In reject mode such requests get a 400. Unset keeps the legacy behavior. MCP routes are not affected Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(proxy): regenerate schema.d.ts and shorten mutable-ok comment for ruff format Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): mark generated session ids so affinity consumers do not pin on them Fireworks x-session-affinity, the router session_affinity pre-call check and the complexity router session pin all read metadata.session_id as a caller-chosen stable key. A missing_session_id: generate id is fresh per request, so it now carries metadata.litellm_session_id_generated and those consumers skip it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/llms/fireworks_ai/common_utils.py | 11 +- litellm/proxy/_types.py | 4 + litellm/proxy/litellm_pre_call_utils.py | 55 ++++++ .../complexity_router/complexity_router.py | 8 +- .../deployment_affinity_check.py | 4 +- .../test_fireworks_ai_chat_transformation.py | 16 ++ .../proxy/test_litellm_pre_call_utils.py | 176 ++++++++++++++++++ .../router_strategy/test_complexity_router.py | 22 ++- .../test_session_id_affinity.py | 43 ++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 11 files changed, 335 insertions(+), 10 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index c7b74e176db..ef9329b9dfc 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1449,6 +1449,7 @@ RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl" CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" +SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated" LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated" LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = ( "Truncation is a DB storage safeguard. " diff --git a/litellm/llms/fireworks_ai/common_utils.py b/litellm/llms/fireworks_ai/common_utils.py index 8c306faa036..ac934ad0cb5 100644 --- a/litellm/llms/fireworks_ai/common_utils.py +++ b/litellm/llms/fireworks_ai/common_utils.py @@ -2,6 +2,7 @@ from typing import Final from httpx import Headers +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -16,16 +17,18 @@ def get_fireworks_session_id(litellm_params: dict) -> str | None: """ Session id to send as `x-session-affinity`, or None when the caller gave none. - Deliberately does not fall back to `litellm_trace_id`: that is generated per - request (`str(uuid.uuid4())` when absent), so using it pins every request to a - different Fireworks node and prompt caching never hits. + Deliberately does not fall back to `litellm_trace_id`, and ignores session ids the + proxy generated for a request that had none: both are per request, so using them + pins every request to a different Fireworks node and prompt caching never hits. """ params: Final = litellm_params + metadata: Final = params.get("metadata") + if isinstance(metadata, dict) and metadata.get(SESSION_ID_GENERATED_METADATA_KEY): + return None for key in ("litellm_session_id", "session_id"): value = params.get(key) if value: return str(value) - metadata: Final = params.get("metadata") if isinstance(metadata, dict): value = metadata.get("session_id") if value: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 2da7ceb2d50..849e54c65aa 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2594,6 +2594,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.", ) + missing_session_id: Literal["generate", "reject"] | None = Field( + None, + description="What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.", + ) enable_public_model_hub: bool = Field( default=False, description="Public model hub for users to see what models they have access to, supported openai params, etc.", diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 20f83085286..1d440448c2f 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -16,6 +16,7 @@ from starlette.datastructures import Headers import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging +from litellm._uuid import uuid from litellm.constants import ( CONSUMED_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, @@ -23,6 +24,7 @@ from litellm.constants import ( OTEL_SERVICE_NAME_METADATA_KEYS, PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + SESSION_ID_GENERATED_METADATA_KEY, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -40,6 +42,7 @@ from litellm.proxy._types import ( AddTeamCallback, CommonProxyErrors, LitellmDataForBackendLLMCall, + LiteLLMRoutes, LitellmUserRoles, ProxyErrorTypes, ProxyException, @@ -47,6 +50,8 @@ from litellm.proxy._types import ( TeamCallbackMetadata, UserAPIKeyAuth, ) +from litellm.proxy.auth.auth_utils import get_request_route +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.callback_utils import ( decrypt_callback_vars, get_metadata_variable_name_from_kwargs, @@ -715,6 +720,50 @@ def _get_anthropic_session_id_from_metadata(metadata: object) -> str | None: return session_id +def _is_llm_inference_route(request: Request) -> bool: + route: Final = get_request_route(request) + return RouteChecks.is_llm_api_route(route=route) and not RouteChecks.check_route_access( + route=route, allowed_routes=LiteLLMRoutes.mcp_routes.value + ) + + +def apply_missing_session_id_policy( + data: dict[str, object], # mutable-ok: stamps session ids in place on the request body the pipeline threads through + _metadata_variable_name: str, + general_settings: Mapping[str, object] | None, + request: Request, +) -> None: + policy: Final = general_settings.get("missing_session_id") if general_settings else None + if policy is None or not _is_llm_inference_route(request): + return + metadata: Final = data.get(_metadata_variable_name) + if not isinstance(metadata, dict): + return + if data.get("litellm_session_id") or metadata.get("session_id"): + return + match policy: + case "generate": + session_id: Final = str(data.get("litellm_trace_id") or metadata.get("trace_id") or uuid.uuid4()) + data["litellm_session_id"] = session_id # rebind-ok: data is an out-param + data.setdefault("litellm_trace_id", session_id) + metadata["session_id"] = session_id + metadata[SESSION_ID_GENERATED_METADATA_KEY] = True + case "reject": + raise ProxyException( + message=( + "Request has no session id. Send an `x-litellm-session-id` header or `metadata.session_id`. " + "Required by `general_settings.missing_session_id: reject`." + ), + type=ProxyErrorTypes.bad_request_error, + param="session_id", + code=400, + ) + case _: + verbose_proxy_logger.warning( + "Ignoring unknown general_settings.missing_session_id=%r; expected 'generate' or 'reject'", policy + ) + + def is_claude_code_user_agent(user_agent: str) -> bool: """Claude Code identifies itself as ``claude-cli/ ...``; the IDE extensions and the Agent SDK run through the same CLI and share that prefix.""" @@ -1818,6 +1867,12 @@ async def add_litellm_data_to_request( data=data, _metadata_variable_name=_metadata_variable_name, ) + apply_missing_session_id_policy( + data=data, + _metadata_variable_name=_metadata_variable_name, + general_settings=general_settings, + request=request, + ) # Expose request headers under the metadata field for guardrails (fixes #17477) if _metadata_variable_name in data and isinstance(data[_metadata_variable_name], dict): diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index d205db90607..430efe339a2 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -26,7 +26,11 @@ from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast from pydantic import BaseModel, create_model from litellm._logging import verbose_router_logger -from litellm.constants import EMPTY_MAPPING, RETURN_RAW_MODEL_NAME_METADATA_KEY +from litellm.constants import ( + EMPTY_MAPPING, + RETURN_RAW_MODEL_NAME_METADATA_KEY, + SESSION_ID_GENERATED_METADATA_KEY, +) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata @@ -2712,7 +2716,7 @@ class ComplexityRouter(CustomLogger): """Resolve a client-supplied session_id.""" for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs): session_id = metadata.get("session_id") - if session_id is not None: + if session_id is not None and not metadata.get(SESSION_ID_GENERATED_METADATA_KEY): return str(session_id) return None diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index b1e9dbdefa8..39d3e25aacb 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -21,7 +21,7 @@ from typing_extensions import TypedDict from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache -from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY +from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger, Span from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import AllMessageValues @@ -265,7 +265,7 @@ class DeploymentAffinityCheck(CustomLogger): @staticmethod def _get_session_id_from_metadata_dict(metadata: dict) -> str | None: session_id: Final = metadata.get("session_id") - if session_id is None: + if session_id is None or metadata.get(SESSION_ID_GENERATED_METADATA_KEY): return None return str(session_id) diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index d7cc89868af..ec8725db5f7 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -8,6 +8,7 @@ import litellm from litellm import get_model_info, supports_reasoning, supports_vision from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -235,6 +236,21 @@ def test_get_fireworks_session_id_prefers_litellm_session_id_over_trace_id(): ) +def test_get_fireworks_session_id_ignores_proxy_generated_session_id(): + """general_settings.missing_session_id: generate stamps a fresh id per request; sending it + as x-session-affinity would pin every request to a different node.""" + assert ( + get_fireworks_session_id( + { + "litellm_session_id": "generated-1", + "litellm_trace_id": "generated-1", + "metadata": {"session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True}, + } + ) + is None + ) + + def test_handle_message_content_with_tool_calls(): config = FireworksAIConfig() message = Message( diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index ee0e2014951..8366e5546a9 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -41,7 +41,9 @@ from litellm.litellm_core_utils.get_provider_specific_headers import ( from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( TRUSTED_CALLBACK_VARS_FIELD, ) +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.types.utils import CredentialItem @@ -7719,3 +7721,177 @@ def test_stamped_model_access_groups_survive_the_litellm_metadata_merge(): } assert get_litellm_metadata_from_kwargs(kwargs)[MODEL_ACCESS_GROUP_METADATA_KEY] == ["tier-a"] + + +def _request_for(path: str) -> MagicMock: + request = MagicMock(spec=Request) + request.scope = {"path": path} + request.url = MagicMock() + request.url.path = path + request.url.__str__.return_value = f"http://localhost{path}" + request.method = "POST" + request.query_params = {} + request.headers = {"Content-Type": "application/json"} + request.client = MagicMock() + request.client.host = "127.0.0.1" + return request + + +def _spend_log_session_id(data: dict[str, object]) -> str: + """Resolve session_id the way LiteLLM_SpendLogs does: standard_logging_payload.trace_id.""" + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.proxy.spend_tracking.spend_tracking_utils import _get_session_id_for_spend_log + + metadata = data["metadata"] + assert isinstance(metadata, dict) + litellm_params = get_litellm_params( + litellm_session_id=str(data["litellm_session_id"]) if "litellm_session_id" in data else None, + litellm_trace_id=str(data["litellm_trace_id"]) if "litellm_trace_id" in data else None, + metadata=metadata, + ) + trace_id = StandardLoggingPayloadSetup.get_standard_logging_payload_trace_id( + logging_obj=SimpleNamespace(litellm_trace_id="per-call-random-trace-id"), + litellm_params=litellm_params, + ) + return _get_session_id_for_spend_log(kwargs={}, standard_logging_payload={"trace_id": trace_id}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_correlation_in_logs", [False, True]) +async def test_missing_session_id_generate_makes_spend_log_and_callback_session_ids_agree( + monkeypatch: pytest.MonkeyPatch, request_correlation_in_logs: bool +): + """Without a session header, SpendLogs.session_id and the metadata.session_id that Langfuse logs + must be the same generated id, so cross-referencing the two by session_id works. The id is marked + as generated so affinity consumers (Fireworks x-session-affinity, router session pins) skip it.""" + monkeypatch.setattr(litellm, "request_correlation_in_logs", request_correlation_in_logs) + data = {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]} + + updated = await add_litellm_data_to_request( + data=data, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "generate"}, + ) + + callback_session_id = updated["metadata"]["session_id"] + assert isinstance(callback_session_id, str) and len(callback_session_id) == 36 + assert _spend_log_session_id(updated) == callback_session_id + assert updated["metadata"][SESSION_ID_GENERATED_METADATA_KEY] is True + assert get_fireworks_session_id( + {"litellm_session_id": updated["litellm_session_id"], "metadata": updated["metadata"]} + ) is None + + +@pytest.mark.asyncio +async def test_missing_session_id_unset_keeps_legacy_divergence(): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + ) + + assert "session_id" not in updated["metadata"] + assert "litellm_session_id" not in updated + assert _spend_log_session_id(updated) == "per-call-random-trace-id" + + +@pytest.mark.asyncio +async def test_missing_session_id_generate_reuses_traceparent_trace_id(): + """A W3C traceparent already decides SpendLogs.session_id, so the callback session id must reuse it.""" + request = _request_for("/v1/chat/completions") + request.headers = {"traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"} + + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "generate"}, + ) + + assert updated["metadata"]["session_id"] == "4bf92f3577b34da6a3ce929d0e0e4736" + assert _spend_log_session_id(updated) == "4bf92f3577b34da6a3ce929d0e0e4736" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["generate", "reject"]) +async def test_missing_session_id_policy_keeps_client_supplied_session_id(policy: str): + request = _request_for("/v1/chat/completions") + request.headers = {"x-litellm-session-id": "client-session-1"} + + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": policy}, + ) + + assert updated["litellm_session_id"] == "client-session-1" + assert updated["metadata"]["session_id"] == "client-session-1" + assert _spend_log_session_id(updated) == "client-session-1" + assert SESSION_ID_GENERATED_METADATA_KEY not in updated["metadata"] + assert ( + get_fireworks_session_id({"litellm_session_id": "client-session-1", "metadata": updated["metadata"]}) + == "client-session-1" + ) + + +@pytest.mark.asyncio +async def test_missing_session_id_reject_accepts_body_metadata_session_id(): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": [], "metadata": {"session_id": "body-session-1"}}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "reject"}, + ) + + assert updated["metadata"]["session_id"] == "body-session-1" + + +@pytest.mark.asyncio +async def test_missing_session_id_reject_returns_400_without_session_id(): + with pytest.raises(ProxyException) as exc_info: + await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "reject"}, + ) + + assert exc_info.value.code == "400" + assert exc_info.value.param == "session_id" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/mcp/", "/mcp/tools", "/key/health"]) +async def test_missing_session_id_policy_skips_non_inference_routes(path: str): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o"}, + request=_request_for(path), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "reject"}, + ) + + assert "session_id" not in updated["metadata"] + + +@pytest.mark.asyncio +async def test_missing_session_id_unknown_value_is_ignored(): + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "messages": []}, + request=_request_for("/v1/chat/completions"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "typo"}, + ) + + assert "session_id" not in updated["metadata"] diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 941d78085e4..d7d02544efb 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -16,7 +16,7 @@ import litellm from litellm import Router from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache -from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY +from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.router_strategy.complexity_router.complexity_router import ( _CLASSIFICATION_CURRENT_MESSAGE_ONLY, _CLASSIFICATION_WITH_CONVERSATION, @@ -4274,6 +4274,26 @@ class TestSessionAffinity: assert first.model == "o1-preview" assert second.model == "gpt-4o-mini" + @pytest.mark.asyncio + async def test_proxy_generated_session_id_never_pins(self, mock_router_instance, session_affinity_config): + """A session id the proxy generated for a request that had none is per request, so + it must not create a pin even with session_affinity enabled.""" + mock_router_instance.cache = DualCache() + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=session_affinity_config, + ) + request_kwargs = {"metadata": {"session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True}} + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs=request_kwargs, messages=self.REASONING_MESSAGE + ) + second = await router.async_pre_routing_hook( + model="test-model", request_kwargs=request_kwargs, messages=self.SIMPLE_MESSAGE + ) + assert first.model == "o1-preview" + assert second.model == "gpt-4o-mini" + @pytest.mark.asyncio async def test_can_be_enabled_to_pin_every_later_turn(self, mock_router_instance, session_affinity_config): """Regression: session_affinity=True is the opt-in, so a shared session_id reuses the diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py b/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py index a3772a276fa..cf48888600e 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_session_id_affinity.py @@ -7,7 +7,7 @@ import json import litellm from litellm.caching.dual_cache import DualCache -from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY +from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, ) @@ -180,6 +180,47 @@ async def test_async_session_id_affinity_priority_over_user_key(): assert filtered[0]["model_info"]["id"] == "deployment-2" +@pytest.mark.asyncio +async def test_proxy_generated_session_id_does_not_pin_a_deployment(): + """A session id the proxy generated for a request that had none is per request, so a + pin stored under it must be ignored and none must be written.""" + cache = DualCache() + callback = DeploymentAffinityCheck( + cache=cache, + ttl_seconds=123, + enable_user_key_affinity=False, + enable_responses_api_affinity=False, + enable_session_id_affinity=True, + ) + healthy_deployments = [ + {"model_name": "model_group", "litellm_params": {"model": "model_1"}, "model_info": {"id": "deployment-1"}}, + {"model_name": "model_group", "litellm_params": {"model": "model_2"}, "model_info": {"id": "deployment-2"}}, + ] + await cache.async_set_cache( + DeploymentAffinityCheck.get_session_affinity_cache_key("model_group", "generated-1", user_key="user1"), + {"model_id": "deployment-2"}, + ) + request_kwargs = { + "metadata": {"user_api_key_hash": "user1", "session_id": "generated-1", SESSION_ID_GENERATED_METADATA_KEY: True} + } + + filtered = await callback.async_filter_deployments( + model="model_group", healthy_deployments=healthy_deployments, messages=[], request_kwargs=request_kwargs + ) + await callback.async_pre_call_deployment_hook( + kwargs={ + "metadata": {**request_kwargs["metadata"], "deployment_model_name": "model_group"}, + "model_info": {"id": "deployment-1"}, + }, + call_type=None, + ) + + assert len(filtered) == 2 + assert await cache.async_get_cache( + DeploymentAffinityCheck.get_session_affinity_cache_key("model_group", "generated-1", user_key="user1") + ) == {"model_id": "deployment-2"} + + MOCK_RESPONSES_API_RESPONSE = { "id": "resp_mock-resp-456", "object": "response", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 90c4f03bf08..491c4fb6a44 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25772,6 +25772,11 @@ export interface components { * @description Number of trusted reverse proxies/load balancers in front of the gateway that append to X-Forwarded-For. When set (and mcp_trusted_proxy_ranges validates the direct peer), the client IP for MCP access control is read this many entries from the right of the chain instead of the spoofable leftmost value, defeating append-style X-Forwarded-For forgery. */ mcp_xff_num_trusted_hops?: number | null; + /** + * Missing Session Id + * @description What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id. + */ + missing_session_id?: ("generate" | "reject") | null; /** * Model List Healthy Only * @description When true, `/models`, `/v1/models/{id}` and `/model/info` hide models whose backing deployments are all unhealthy, for every caller, without needing `healthy_only=true` per request. Requires `background_health_checks: true`, and keeps deployment health state cached without turning on `enable_health_check_routing`, so routing is unaffected. With no health state nothing is hidden. Hiding is presentation-only, a hidden model can still be called. From 78ff5ac9cd144770f9e53931d190ca2d480d9d98 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 2 Sep 2026 18:31:25 -0700 Subject: [PATCH 3/3] feat(router): arm safeguard-refusal fallback on generic chains when no content-policy list exists (#39274) --- litellm/router.py | 30 ++++- .../router_utils/fallback_event_handlers.py | 32 ++++++ ...test_router_anthropic_messages_fallback.py | 106 ++++++++++++++++++ 3 files changed, 166 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 0af514fe8a2..303b22c9484 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -150,8 +150,10 @@ from litellm.router_utils.fallback_event_handlers import ( _check_non_standard_fallback_format, clear_pre_routing_selection, fallback_lookup_groups, + fallbacks_disabled_for_request, get_fallback_model_group_for_lookup_groups, get_pre_routing_selection, + record_disable_fallbacks, record_pre_routing_selection, run_async_fallback, ) @@ -5193,7 +5195,7 @@ class Router: if not has_generated_content and error_event is None else None ) - if refusal_stop_details is not None and self._has_content_policy_fallback(model, initial_kwargs): + if refusal_stop_details is not None and self._refusal_fallback_available(model, initial_kwargs): refusal_error = safeguard_refusal_error(model=model, stop_details=refusal_stop_details) raise MidStreamFallbackError( message=refusal_error.message, @@ -7266,6 +7268,7 @@ class Router: _fallback_metadata["original_model_group"] = model_group include_fallback_errors: Final = kwargs.get("include_fallback_errors", False) is True disable_fallbacks: Final[bool | None] = kwargs.pop("disable_fallbacks", False) + record_disable_fallbacks(kwargs, disable_fallbacks is True) fallbacks: Final[list | None] = kwargs.get("fallbacks", self.fallbacks) context_window_fallbacks: list | None = kwargs.get("context_window_fallbacks", self.context_window_fallbacks) content_policy_fallbacks: list | None = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) @@ -8131,6 +8134,29 @@ class Router: ) return False + def _refusal_fallback_available(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: + """ + Whether a safeguard refusal can actually be recovered by the dispatcher. A configured + content-policy list is authoritative; with none configured at all, the dispatcher falls + through to the generic fallbacks lookup, so the gate mirrors that reachability and arms + on a resolving generic chain (tier first, then the requested group, then "*"). + """ + if fallbacks_disabled_for_request(kwargs): + return False + content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) + if content_policy_fallbacks is not None: + return self._has_content_policy_fallback(model_group, kwargs) + if self._has_default_fallbacks(): + return True + fallbacks: Final = kwargs.get("fallbacks", self.fallbacks) + if fallbacks is None: + return False + resolved, _ = get_fallback_model_group_for_lookup_groups( + fallbacks=fallbacks, + lookup_groups=fallback_lookup_groups(kwargs, model_group), + ) + return resolved is not None + def _should_raise_content_policy_error(self, model: str, response: ModelResponse, kwargs: dict) -> bool: """ Determines if a content policy error should be raised. @@ -8162,7 +8188,7 @@ class Router: return False if get_safeguard_refusal_stop_details(response) is None: return False - return self._has_content_policy_fallback(model, kwargs) + return self._refusal_fallback_available(model, kwargs) def _get_healthy_deployments(self, model: str, parent_otel_span: Span | None): _all_deployments: list = [] diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index f7855cb38ff..601b32c4386 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -263,6 +263,38 @@ def get_pre_routing_selection(kwargs: Mapping[str, Any]) -> str | None: return next((selected for selected in selections if isinstance(selected, str) and selected), None) +DISABLE_FALLBACKS_METADATA_KEY: Final = "_disable_fallbacks" + + +def record_disable_fallbacks(request_kwargs: Mapping[str, Any] | None, disabled: bool) -> None: + """ + Write-or-clear the request's disable_fallbacks verdict into the router-internal metadata + bucket. The wrapper pops the raw kwarg before any downstream frame runs, so the refusal + gate (which decides whether to convert a refusal into a recoverable error) needs this + carrier to know recovery is impossible. + """ + from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs + + if request_kwargs is None: + return + bucket: Final = request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs)) + if not isinstance(bucket, dict): + return + if disabled: + bucket[DISABLE_FALLBACKS_METADATA_KEY] = True + else: + bucket.pop(DISABLE_FALLBACKS_METADATA_KEY, None) + + +def fallbacks_disabled_for_request(kwargs: Mapping[str, Any]) -> bool: + """True when this request opted out of fallbacks, read from the raw kwarg (pre-pop + snapshots keep it) or the router-internal bucket the wrapper stamps after popping it.""" + if kwargs.get("disable_fallbacks") is True: + return True + buckets: Final = (kwargs.get(name) for name in _ROUTER_METADATA_BUCKETS) + return any(isinstance(bucket, dict) and bucket.get(DISABLE_FALLBACKS_METADATA_KEY) is True for bucket in buckets) + + def fallback_lookup_groups(kwargs: Mapping[str, Any], model_group: str | None) -> tuple[str, ...]: """ Ordered keys for resolving a fallback chain: the tier a pre-routing hook selected wins, diff --git a/tests/router_unit_tests/test_router_anthropic_messages_fallback.py b/tests/router_unit_tests/test_router_anthropic_messages_fallback.py index 0c4d1dfc21e..4812d199c06 100644 --- a/tests/router_unit_tests/test_router_anthropic_messages_fallback.py +++ b/tests/router_unit_tests/test_router_anthropic_messages_fallback.py @@ -338,6 +338,112 @@ def test_record_pre_routing_selection_writes_only_the_internal_bucket(): assert kwargs["metadata"] == {"user_id": "u1"} +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_generic_only_row_recovers_safeguard_refusal(stream): + """With no content-policy list configured, a generic fallback row covers safeguard refusals, + so the dashboard's generic fallbacks work without config-only content_policy rows.""" + fake = FakeAnthropicUpstream() + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], fallbacks=[{"fable-tier": ["opus-target"]}]) + + with fake.install(): + response = await router.aanthropic_messages( + model="fable-tier", max_tokens=16, stream=stream, messages=[{"role": "user", "content": "hi"}] + ) + body = await _collect(response) if stream else response + + if stream: + assert b'"refusal"' not in body + assert b"text_delta" in body + else: + assert body["stop_reason"] == "end_turn" + assert len(fake.calls) == 2 + assert "claude-opus-5" in fake.calls[1] + + +@pytest.mark.asyncio +async def test_configured_content_policy_list_stays_authoritative_over_generic_rows(): + fake = FakeAnthropicUpstream() + router = Router( + model_list=[FABLE_TIER, OPUS_TARGET], + fallbacks=[{"fable-tier": ["opus-target"]}], + content_policy_fallbacks=[{"unrelated-group": ["opus-target"]}], + ) + + with fake.install(): + response = await router.aanthropic_messages( + model="fable-tier", max_tokens=16, messages=[{"role": "user", "content": "hi"}] + ) + + assert response["stop_reason"] == "refusal" + assert len(fake.calls) == 1 + + +def test_refusal_fallback_available_arms_on_generic_rows_only_without_content_policy(): + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], fallbacks=[{"tier-group": ["opus-target"]}]) + stamped = {"litellm_metadata": {PRE_ROUTING_SELECTED_MODEL_KEY: "tier-group"}} + + assert router._refusal_fallback_available("router-group", stamped) is True + assert router._refusal_fallback_available("router-group", {}) is False + assert router._refusal_fallback_available("router-group", {"content_policy_fallbacks": [{"other": ["x"]}]}) is False + + +def test_chat_content_filter_gate_unchanged_by_generic_rows(): + """The generic-row arming is scoped to /v1/messages safeguard refusals; the chat surface's + content_filter gate keeps its long-standing content-policy-only semantics.""" + from litellm.types.utils import Choices, ModelResponse + + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], fallbacks=[{"fable-tier": ["opus-target"]}]) + response = ModelResponse(choices=[Choices(finish_reason="content_filter")]) + + assert router._should_raise_content_policy_error(model="fable-tier", response=response, kwargs={}) is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_disable_fallbacks_returns_the_refusal_instead_of_raising(stream): + """A request that opted out of fallbacks must receive the provider's refusal response, + never a ContentPolicyViolationError the dispatcher refuses to recover.""" + fake = FakeAnthropicUpstream() + router = Router(model_list=[FABLE_TIER, OPUS_TARGET], fallbacks=[{"fable-tier": ["opus-target"]}]) + + with fake.install(): + response = await router.aanthropic_messages( + model="fable-tier", + max_tokens=16, + stream=stream, + disable_fallbacks=True, + messages=[{"role": "user", "content": "hi"}], + ) + body = await _collect(response) if stream else response + + if stream: + assert b'"stop_reason": "refusal"' in body + else: + assert body["stop_reason"] == "refusal" + assert len(fake.calls) == 1 + + +@pytest.mark.asyncio +async def test_disable_fallbacks_beats_a_content_policy_row_too(): + fake = FakeAnthropicUpstream() + router = Router( + model_list=[FABLE_TIER, OPUS_TARGET], + content_policy_fallbacks=[{"fable-tier": ["opus-target"]}], + ) + + with fake.install(): + response = await router.aanthropic_messages( + model="fable-tier", + max_tokens=16, + disable_fallbacks=True, + messages=[{"role": "user", "content": "hi"}], + ) + + assert response["stop_reason"] == "refusal" + assert len(fake.calls) == 1 + + def test_refusal_gate_keys_on_pre_routing_tier_stamp(): router = _router(content_policy_fallbacks=[{"tier-group": ["opus-target"]}])