diff --git a/.github/assets/scaledown/provider-after.jpg b/.github/assets/scaledown/provider-after.jpg new file mode 100644 index 00000000000..bcf1c59c96f Binary files /dev/null and b/.github/assets/scaledown/provider-after.jpg differ diff --git a/.github/assets/scaledown/provider-before.jpg b/.github/assets/scaledown/provider-before.jpg new file mode 100644 index 00000000000..a1b89ae13ca Binary files /dev/null and b/.github/assets/scaledown/provider-before.jpg differ diff --git a/.github/workflows/publish-lint-base-counts.yml b/.github/workflows/publish-lint-base-counts.yml index 5623ab82b90..bbe9cadc893 100644 --- a/.github/workflows/publish-lint-base-counts.yml +++ b/.github/workflows/publish-lint-base-counts.yml @@ -35,6 +35,8 @@ jobs: script: scripts/type_discipline_gate.py - checker: test-quality script: scripts/test_quality_gate.py + - checker: openapi-docs + script: scripts/openapi_docs_gate.py - checker: basedpyright script: scripts/type_check_gate.py @@ -60,16 +62,24 @@ jobs: uses: ./.github/actions/cache-cargo-build - name: Cache Prisma binaries - if: matrix.checker == 'basedpyright' + if: matrix.checker == 'basedpyright' || matrix.checker == 'openapi-docs' uses: ./.github/actions/cache-prisma-binaries - # The three source scanners only need the pinned dev tools (ruff and the + # The source scanners only need the pinned dev tools (ruff and the # stdlib checkers), the same versions test-linting.yml's python job runs. - name: Install the dev tools - if: matrix.checker != 'basedpyright' + if: matrix.checker != 'basedpyright' && matrix.checker != 'openapi-docs' run: | uv sync --frozen --only-group dev --no-install-project + # The OpenAPI gate imports the proxy app, so it needs the same install the + # test-linting.yml python job uses, Prisma client included. + - name: Install the proxy + if: matrix.checker == 'openapi-docs' + run: | + uv sync --frozen --group proxy-dev --group e2e-dev --extra proxy + uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + - name: Emit ${{ matrix.checker }} counts for HEAD if: matrix.checker != 'basedpyright' run: | diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index e85aaacd3f8..2d0766e0468 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -156,6 +156,14 @@ jobs: run: | uv run --no-sync python scripts/test_quality_gate.py --base "$GATE_BASE_SHA" + - name: Check OpenAPI docs (POST routes without a request schema, description or 200 schema, delta vs merge-base counts) + if: steps.changes.outputs.decision != 'skip' + env: + GH_TOKEN: ${{ github.token }} + run: | + uv sync --frozen --group proxy-dev --group e2e-dev --extra proxy + uv run --no-sync python scripts/openapi_docs_gate.py --base "$GATE_BASE_SHA" + - name: Print OpenAI version if: steps.changes.outputs.decision != 'skip' run: | diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 7f01b38ddc8..faa02c8f1a9 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -327,6 +327,7 @@ jobs: tests/unit/llms/sagemaker tests/unit/llms/sail tests/unit/llms/sambanova + tests/unit/llms/scaledown tests/unit/llms/sap tests/unit/llms/scaleway tests/unit/llms/searchapi diff --git a/AGENTS.md b/AGENTS.md index 0254893554d..7f8e9cedc26 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,7 +54,7 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a Python max line length is 120, not 88 -The four lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/test_quality_gate.py`, `scripts/type_check_gate.py`) compare each rule's codebase count on your branch against the count at its merge-base with the default branch, and a rule may not grow. There are no budget files to edit or ratchet: when a gate fails, fix the violations the branch introduced or remove at least as many of that rule elsewhere in the tree. The one exception is `reportAny` / `reportExplicitAny`, which share a fixed codebase-wide cap in `ANY_CAPS` in `scripts/type_check_gate.py` because Any spreads past the lines you touch. A branch may add Anys while the total stays under that cap. Only lower `ANY_CAPS`, and only in its own PR to the default branch, never raise it on a feature branch +The five lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/test_quality_gate.py`, `scripts/openapi_docs_gate.py`, `scripts/type_check_gate.py`) compare each rule's codebase count on your branch against the count at its merge-base with the default branch, and a rule may not grow. There are no budget files to edit or ratchet: when a gate fails, fix the violations the branch introduced or remove at least as many of that rule elsewhere in the tree. The one exception is `reportAny` / `reportExplicitAny`, which share a fixed codebase-wide cap in `ANY_CAPS` in `scripts/type_check_gate.py` because Any spreads past the lines you touch. A branch may add Anys while the total stays under that cap. Only lower `ANY_CAPS`, and only in its own PR to the default branch, never raise it on a feature branch `make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice diff --git a/Makefile b/Makefile index 88cce280065..2a96a8a5e93 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ test-rust-extension rust-sqlx-prepare \ info lint lint-inner lint-dev lint-checks format \ lint-basedpyright lint-e2e-basedpyright lint-type-discipline \ - lint-ruff-strict lint-gate lint-test-quality \ + lint-ruff-strict lint-gate lint-test-quality lint-openapi-docs \ install-dev install-proxy-dev install-test-deps install-hooks \ install-helm-unittest check-circular-imports check-import-safety check check-inner pre-commit \ lint-install lint-fetch-base bootstrap @@ -35,6 +35,7 @@ help: @echo " make lint-ruff-strict - Gate each strict ruff rule's codebase total against its merge-base count" @echo " make lint-gate - Strict ruff gate in CI-parity mode (fetches the default branch, simulates the merge)" @echo " make lint-test-quality - Gate the test suite's TQ counts against their merge-base counts" + @echo " make lint-openapi-docs - Gate undocumented POST routes in the proxy OpenAPI document against their merge-base counts" @echo " make check-circular-imports - Check for circular imports" @echo " make check-import-safety - Check import safety" @echo " make test - Run all tests" @@ -219,6 +220,12 @@ lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) lint-test-quality: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) $(UV_RUN) python scripts/test_quality_gate.py --base "$(BASE_REF)" +# OpenAPI docs gate (POST routes without a request body schema, a description or a +# 200 schema in app.openapi()), counted the same delta-vs-base way so a new endpoint +# has to show up documented in Swagger UI. +lint-openapi-docs: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) + $(UV_RUN) python scripts/openapi_docs_gate.py --base "$(BASE_REF)" + lint-format: format-check lint-ruff-strict: install-dev @@ -250,7 +257,7 @@ lint-inner: lint-install @base_ref=$$($(RESOLVE_BASE)) && \ $(MAKE) BASE_REF="$$base_ref" -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks -lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-test-quality lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety +lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-test-quality lint-openapi-docs lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety # Faster linting for local development (only checks changed code) lint-dev: lint-format-changed check-circular-imports check-import-safety diff --git a/README.md b/README.md index 141e8232de8..9987f13d127 100644 --- a/README.md +++ b/README.md @@ -407,6 +407,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | | | [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ | | | | | | | | | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | | +| [ScaleDown (`scaledown`)](https://docs.litellm.ai/docs/providers/scaledown) | ✅ | | | | | | | | | | | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | | | [Strands Decider (`strands_decider`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | | | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/litellm/__init__.py b/litellm/__init__.py index a6bbd132cbf..554b37f11d4 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -674,6 +674,7 @@ nvidia_nim_models: Set = set() nvidia_riva_models: Set = set() soniox_models: Set = set() sambanova_models: Set = set() +scaledown_models: Final[Set[str]] = set() # Price reloads update this registry through existing references. sambanova_embedding_models: Set = set() novita_models: Set = set() assemblyai_models: Set = set() @@ -756,7 +757,7 @@ def is_openai_finetune_model(key: str) -> bool: return key.startswith("ft:") and not key.count(":") > 1 -def _populate_provider_model_sets(model_cost_map: Dict) -> None: +def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None: for key, value in model_cost_map.items(): if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key): open_ai_chat_completion_models.add(key) @@ -918,6 +919,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None: soniox_models.add(key) elif value.get("litellm_provider") == "sambanova": sambanova_models.add(key) + elif value.get("litellm_provider") == "scaledown": + scaledown_models.add(key) elif value.get("litellm_provider") == "sambanova-embedding-models": sambanova_embedding_models.add(key) elif value.get("litellm_provider") == "novita": @@ -1106,6 +1109,7 @@ model_list = list( | nvidia_riva_models | soniox_models | sambanova_models + | scaledown_models | azure_text_models | novita_models | assemblyai_models @@ -1215,6 +1219,7 @@ def _build_models_by_provider() -> dict: "nvidia_riva": nvidia_riva_models, "soniox": soniox_models, "sambanova": sambanova_models | sambanova_embedding_models, + "scaledown": scaledown_models, "novita": novita_models, "nebius": nebius_models | nebius_embedding_models, "aiml": aiml_models, @@ -2054,6 +2059,9 @@ if TYPE_CHECKING: from .llms.featherless_ai.chat.transformation import ( FeatherlessAIConfig as FeatherlessAIConfig, ) + from .llms.scaledown.chat.transformation import ( + ScaleDownChatConfig as ScaleDownChatConfig, + ) from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig from .llms.nadir.chat.transformation import NadirConfig as NadirConfig from .llms.baseten.chat import BasetenConfig as BasetenConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 9caba6a060a..ce69de3b240 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -277,6 +277,7 @@ LLM_CONFIG_NAMES: Final = ( "NvidiaNimConfig", "NvidiaNimEmbeddingConfig", "FeatherlessAIConfig", + "ScaleDownChatConfig", "CerebrasConfig", "NadirConfig", "BasetenConfig", @@ -1103,6 +1104,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.featherless_ai.chat.transformation", "FeatherlessAIConfig", ), + "ScaleDownChatConfig": ( + ".llms.scaledown.chat.transformation", + "ScaleDownChatConfig", + ), "CerebrasConfig": (".llms.cerebras.chat", "CerebrasConfig"), "NadirConfig": (".llms.nadir.chat.transformation", "NadirConfig"), "BasetenConfig": (".llms.baseten.chat", "BasetenConfig"), diff --git a/litellm/_service_logger.py b/litellm/_service_logger.py index 24625382d0c..6385aa96272 100644 --- a/litellm/_service_logger.py +++ b/litellm/_service_logger.py @@ -16,7 +16,7 @@ from .types.services import ServiceLoggerPayload, ServiceTypes if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth Span = _Span | Any OTELClass = OpenTelemetry diff --git a/litellm/constants.py b/litellm/constants.py index a339fb76976..38d16d61d7b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -234,6 +234,7 @@ MCP_CLIENT_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_CLIENT_TIMEOUT", "60.0" MCP_TOOL_LISTING_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "30.0")) MCP_METADATA_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_METADATA_TIMEOUT", "10.0")) MCP_HEALTH_CHECK_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", "10.0")) +MCP_UPSTREAM_SESSION_SHUTDOWN_TIMEOUT_SECONDS: Final = 5.0 MCP_TOOL_LISTING_MAX_PAGES: Final = 1000 MCP_GATEWAY_SESSION_ID_PREFIX_LENGTH: Final = 8 MCP_BYOK_CREDENTIAL_CACHE_TTL_SECONDS: Final = 60 @@ -825,6 +826,7 @@ LITELLM_CHAT_PROVIDERS: Final = [ "lemonade", "docker_model_runner", "amazon_nova", + "scaledown", ] # Resolving these providers runs an OAuth device flow (their provider info IS the login), so any diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 729a6acb20d..9c5608076a2 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -8,12 +8,12 @@ import hashlib import json import os import time -from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Sequence -from contextlib import AbstractAsyncContextManager +from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Mapping, Sequence +from contextlib import AbstractAsyncContextManager, AsyncExitStack, nullcontext from functools import partial from importlib.metadata import version from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeAlias, TypeVar, cast +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, TypeVar, cast, overload import anyio import httpx2 @@ -21,26 +21,42 @@ from httpx2._client import UseClientDefault from httpx2._types import AuthTypes from mcp import ClientSession, MCPError, ReadResourceResult, Resource, StdioServerParameters from mcp.client._input_required import run_input_required_driver +from mcp.client.session import ClientRequestContext, ElicitationFnT, LoggingFnT, SamplingFnT from mcp.client.sse import sse_client from mcp.client.stdio import stdio_client from mcp.client.streamable_http import streamable_http_client from mcp.shared._stream_protocols import ReadStream, WriteStream from mcp.shared.message import SessionMessage +from typing_extensions import ReadOnly, TypedDict, assert_never _TransportStreams: TypeAlias = tuple[ ReadStream[SessionMessage | Exception], WriteStream[SessionMessage], ] _TransportContext: TypeAlias = AbstractAsyncContextManager[_TransportStreams] +_SessionCallbackName: TypeAlias = Literal["sampling_callback", "elicitation_callback", "logging_callback"] + + +class _SessionCallbackMap(TypedDict): + sampling_callback: ReadOnly[SamplingFnT | None] + elicitation_callback: ReadOnly[ElicitationFnT | None] + logging_callback: ReadOnly[LoggingFnT | None] from mcp.types import ( + CONNECTION_CLOSED, METHOD_NOT_FOUND, REQUEST_TIMEOUT, CacheableResult, ClientCapabilities, + CreateMessageRequestParams, + CreateMessageResult, + CreateMessageResultWithTools, DiscoverResult, ElicitationCapability, + ElicitRequestParams, + ElicitResult, + ErrorData, FormElicitationCapability, GetPromptRequestParams, GetPromptResult, @@ -58,6 +74,7 @@ from mcp.types import ( ListResourceTemplatesResult, ListToolsRequest, ListToolsResult, + LoggingMessageNotificationParams, PaginatedRequestParams, Prompt, ResourceTemplate, @@ -302,6 +319,12 @@ class MCPSigV4Auth(httpx2.Auth): raise ImportError("Missing botocore to use AWS SigV4 authentication. Run 'pip install boto3'.") self.service_name = aws_service_name or "bedrock-agentcore" self.region_name = aws_region_name or "us-east-1" + self.identity: Final[tuple[str, str, str | None, str | None]] = ( + self.service_name, + self.region_name, + aws_role_name, + aws_access_key_id, + ) # Note: os.environ/ prefixed values are already resolved by # ProxyConfig._check_for_os_environ_vars() at config load time. # Values arrive here as plain strings. @@ -346,7 +369,7 @@ class MCPSigV4Auth(httpx2.Auth): import time import boto3 - from botocore.credentials import Credentials + from botocore.credentials import RefreshableCredentials session_name: Final = aws_session_name or f"litellm-mcp-{int(time.time())}" sts_kwargs: Final[dict] = {"region_name": aws_region_name} @@ -356,15 +379,23 @@ class MCPSigV4Auth(httpx2.Auth): if aws_session_token: sts_kwargs["aws_session_token"] = aws_session_token sts_client: Final = boto3.client("sts", **sts_kwargs) - sts_response: Final = sts_client.assume_role( - RoleArn=aws_role_name, - RoleSessionName=session_name, - ) - sts_creds: Final = sts_response["Credentials"] - return Credentials( - access_key=sts_creds["AccessKeyId"], - secret_key=sts_creds["SecretAccessKey"], - token=sts_creds["SessionToken"], + + def assume() -> Mapping[str, str]: + sts_creds: Final = sts_client.assume_role( + RoleArn=aws_role_name, + RoleSessionName=session_name, + )["Credentials"] + return MappingProxyType( + { + "access_key": sts_creds["AccessKeyId"], + "secret_key": sts_creds["SecretAccessKey"], + "token": sts_creds["SessionToken"], + "expiry_time": sts_creds["Expiration"].isoformat(), + } + ) + + return RefreshableCredentials.create_from_metadata( + metadata=dict(assume()), refresh_using=assume, method="sts-assume-role" ) def auth_flow(self, request: httpx2.Request) -> Generator[httpx2.Request, httpx2.Response, None]: @@ -390,6 +421,19 @@ class MCPSigV4Auth(httpx2.Auth): yield request +def _notify_stream_error( + callback: Callable[[asyncio.Future[Exception]], None] | None, + stream_error: asyncio.Future[Exception], +) -> None: + if callback is not None: + callback(stream_error) + + +def _notify_cleanup(callback: Callable[[], None] | None) -> None: + if callback is not None: + callback() + + class MCPClient: """ MCP Client supporting: @@ -443,12 +487,37 @@ class MCPClient: self._sampling_callback: Callable | None = sampling_callback self._elicitation_callback: ElicitationCallback | None = elicitation_callback self._logging_callback: Callable | None = logging_callback + self._session_callbacks_override: _SessionCallbackMap | None = None # handle the basic auth value if provided if auth_value: self.update_auth_value(auth_value) + def session_callbacks(self) -> _SessionCallbackMap: + return { + "sampling_callback": self._sampling_callback, + "elicitation_callback": self._elicitation_callback, + "logging_callback": self._logging_callback, + } + + def session_settings(self) -> tuple[MCPTransport, MCPUpstreamProtocol, float, bool, bool, bool]: + callbacks: Final = self.session_callbacks() + return ( + self.transport_type, + self.protocol_version, + self.timeout, + callbacks["sampling_callback"] is not None, + callbacks["elicitation_callback"] is not None, + callbacks["logging_callback"] is not None, + ) + + def route_session_callbacks(self, callbacks: _SessionCallbackMap) -> None: + self._session_callbacks_override = callbacks + async def discovery_auth_fingerprint(self) -> str: - return self._hash_discovery_auth(await self.prepare_request_auth()) + return self._hash_discovery_auth(await self.prepare_request_auth(), self._aws_identity()) + + def _aws_identity(self) -> tuple[str, str, str | None, str | None] | None: + return self._aws_auth.identity if isinstance(self._aws_auth, MCPSigV4Auth) else None async def prepare_request_auth(self) -> httpx2.Request: """Preview the authenticated request without sending it, closing the auth flow afterwards.""" @@ -465,8 +534,10 @@ class MCPClient: await flow.aclose() @staticmethod - def _hash_discovery_auth(request: httpx2.Request) -> str: - material: Final = json.dumps((str(request.url), tuple(sorted(request.headers.multi_items())))) + def _hash_discovery_auth( + request: httpx2.Request, aws_identity: tuple[str, str, str | None, str | None] | None + ) -> str: + material: Final = json.dumps((str(request.url), tuple(sorted(request.headers.multi_items())), aws_identity)) return hashlib.sha256(material.encode()).hexdigest() def _create_transport_context( @@ -605,6 +676,9 @@ class MCPClient: transport_ctx: _TransportContext, operation: Callable[[ClientSession], Awaitable[TSessionResult]], http_client: httpx2.AsyncClient | None = None, + *, + on_stream_error: Callable[[asyncio.Future[Exception]], None] | None = None, + on_cleanup: Callable[[], None] | None = None, ) -> TSessionResult: """ Execute an operation within a transport and session context. @@ -623,6 +697,7 @@ class MCPClient: read_stream: Final = transport[0] write_stream: Final = transport[1] stream_error: Final[asyncio.Future[Exception]] = asyncio.get_running_loop().create_future() + _notify_stream_error(on_stream_error, stream_error) async def receive_message( message: ServerNotification | Exception, @@ -634,6 +709,11 @@ class MCPClient: # The SDK closes pending requests when its message handler raises. raise RuntimeError("MCP response stream failed") + session_kwargs: Final = ( + self._session_callbacks_override + if self._session_callbacks_override is not None + else self.session_callbacks() + ) # The SDK drops a response stream that ends without a JSON-RPC reply, so nothing else # ever fails the request. session_ctx: Final = ClientSession( @@ -641,9 +721,7 @@ class MCPClient: write_stream, read_timeout_seconds=self.timeout, message_handler=receive_message, - sampling_callback=self._sampling_callback, - elicitation_callback=self._elicitation_callback, - logging_callback=self._logging_callback, + **session_kwargs, ) session: Final = await session_ctx.__aenter__() try: @@ -659,6 +737,7 @@ class MCPClient: raise stream_error.result() raise finally: + _notify_cleanup(on_cleanup) cleanup_scope.shield = True cleanup_scope.deadline = anyio.current_time() + 5 try: @@ -671,6 +750,7 @@ class MCPClient: in_flight_error = e raise finally: + _notify_cleanup(on_cleanup) cleanup_scope.shield = True cleanup_scope.deadline = min(cleanup_scope.deadline, anyio.current_time() + 5) try: @@ -703,6 +783,8 @@ class MCPClient: operation: Callable[[ClientSession], Awaitable[TSessionResult]], *, quiet_on_error: bool = False, + on_stream_error: Callable[[asyncio.Future[Exception]], None] | None = None, + on_cleanup: Callable[[], None] | None = None, ) -> TSessionResult: """Open a session, run the provided coroutine, and clean up. @@ -714,7 +796,18 @@ class MCPClient: try: self._last_initialize_instructions = None transport_ctx, http_client = self._create_transport_context() - result: Final = await self._execute_session_operation(transport_ctx, operation, http_client=http_client) + execution: Final = ( + self._execute_session_operation( + transport_ctx, + operation, + http_client=http_client, + on_stream_error=on_stream_error, + on_cleanup=on_cleanup, + ) + if on_stream_error is not None or on_cleanup is not None + else self._execute_session_operation(transport_ctx, operation, http_client=http_client) + ) + result: Final = await execution except Exception as e: read_timeout: Final = as_mcp_read_timeout(e) if read_timeout is not None: @@ -741,6 +834,12 @@ class MCPClient: await anyio.lowlevel.checkpoint_if_cancelled() return result + def open_persistent_session( + self, + admission: Callable[[], AbstractAsyncContextManager[None]] | None = None, + ) -> "PersistentMCPSession": + return PersistentMCPSession(self, admission) + def update_auth_value(self, mcp_auth_value: str | dict[str, str]) -> None: """ Set the authentication header for the MCP client. @@ -972,10 +1071,15 @@ class MCPClient: host_progress_callback: Callable | None = None, raise_on_error: bool = False, allow_input_required: bool = False, + persistent_session: "PersistentMCPSession | None" = None, ) -> MCPCallToolResult | InputRequiredResult: """ Call an MCP Tool. + persistent_session runs the call inside an already-open upstream session (one + upstream mcp-session-id shared by every call of a stateful gateway session) instead of + the per-call initialize + teardown that run_with_session performs. + Args: raise_on_error: When True, re-raise the underlying exception instead of returning an ``isError=True`` result. The token-exchange (OBO) tool-call path uses this to detect @@ -1030,7 +1134,15 @@ class MCPClient: ) try: - tool_result: Final = await self.run_with_session(_call_tool_operation, quiet_on_error=raise_on_error) + tool_result: Final = ( + await self.run_with_session(_call_tool_operation, quiet_on_error=raise_on_error) + if persistent_session is None + else await persistent_session.run( + _call_tool_operation, + quiet_on_error=raise_on_error, + callbacks=self.session_callbacks(), + ) + ) verbose_logger.info("MCP client tool call '%s' completed successfully", call_tool_request_params.name) return tool_result except asyncio.CancelledError: @@ -1395,3 +1507,256 @@ class MCPClient: "the MCP server may have crashed, disconnected, or timed out." ) raise + + +_PendingOperation: TypeAlias = tuple[ + Callable[[ClientSession], Awaitable[object]], + asyncio.Future[object], + _SessionCallbackMap | None, +] +_MAX_PENDING_OPERATIONS: Final = 64 + + +def _ends_session(outcome: BaseException) -> bool: + if isinstance( + outcome, + (httpx2.HTTPError, OSError, anyio.BrokenResourceError, anyio.ClosedResourceError, anyio.EndOfStream), + ): + return True + return isinstance(outcome, MCPError) and outcome.error.code == CONNECTION_CLOSED + + +def _deliver_persistent_outcome( + future: asyncio.Future[object], + outcome: object, + stream_error: asyncio.Future[Exception] | None, +) -> bool: + mapped_outcome: Final = ( + stream_error.result() + if isinstance(outcome, MCPError) and stream_error is not None and stream_error.done() + else outcome + ) + if not future.done(): + if isinstance(mapped_outcome, Exception): + future.set_exception(mapped_outcome) + elif isinstance(mapped_outcome, BaseException): + future.set_exception(RuntimeError("upstream MCP operation was cancelled")) + else: + future.set_result(mapped_outcome) + return ( + isinstance(outcome, BaseException) + and _ends_session(outcome) + or isinstance(mapped_outcome, BaseException) + and _ends_session(mapped_outcome) + or stream_error is not None + and stream_error.done() + ) + + +class UpstreamSessionClosedError(RuntimeError): + def __init__(self) -> None: + super().__init__("upstream MCP session closed") + + +class PersistentMCPSession: + """One upstream MCP session kept open across operations. + + The SDK transport owns anyio cancel scopes that must be entered and exited by the same + task, so a dedicated task holds the session open and serves queued operations in order. + """ + + def __init__( + self, + client: MCPClient, + admission: Callable[[], AbstractAsyncContextManager[None]] | None = None, + ) -> None: + loop: Final = asyncio.get_running_loop() + self._client: Final = client + self._admission: Final = admission + self._queue: Final[asyncio.Queue[_PendingOperation]] = asyncio.Queue(maxsize=_MAX_PENDING_OPERATIONS) + self._ready: Final[asyncio.Future[None]] = loop.create_future() + self._active: asyncio.Future[object] | None = None + self._stream_error: asyncio.Future[Exception] | None = None + self._retiring: bool = False + self._ended: bool = False + self._initial_callbacks: Final[_SessionCallbackMap] = client.session_callbacks() + self._current_callbacks: _SessionCallbackMap = self._initial_callbacks + delegated_callbacks: Final[_SessionCallbackMap] = { + "sampling_callback": ( + self._delegate("sampling_callback") + if self._initial_callbacks["sampling_callback"] is not None + else None + ), + "elicitation_callback": ( + self._delegate("elicitation_callback") + if self._initial_callbacks["elicitation_callback"] is not None + else None + ), + "logging_callback": ( + self._delegate("logging_callback") if self._initial_callbacks["logging_callback"] is not None else None + ), + } + client.route_session_callbacks(delegated_callbacks) + self._task: Final = loop.create_task(self._serve()) + + @overload + def _delegate(self, name: Literal["sampling_callback"]) -> SamplingFnT: ... + + @overload + def _delegate(self, name: Literal["elicitation_callback"]) -> ElicitationFnT: ... + + @overload + def _delegate(self, name: Literal["logging_callback"]) -> LoggingFnT: ... + + def _delegate(self, name: _SessionCallbackName) -> SamplingFnT | ElicitationFnT | LoggingFnT: + match name: + case "sampling_callback": + + async def sampling_callback( + context: ClientRequestContext, params: CreateMessageRequestParams + ) -> CreateMessageResult | CreateMessageResultWithTools | ErrorData: + current_callback: Final = self._current_callbacks.get(name) + initial_callback: Final = self._initial_callbacks.get(name) + callback: Final = current_callback if current_callback is not None else initial_callback + if callback is None: + raise RuntimeError("sampling callback is unavailable") + return await callback(context, params) + + return sampling_callback + case "elicitation_callback": + + async def elicitation_callback( + context: ClientRequestContext, params: ElicitRequestParams + ) -> ElicitResult | ErrorData: + current_callback: Final = self._current_callbacks.get(name) + initial_callback: Final = self._initial_callbacks.get(name) + callback: Final = current_callback if current_callback is not None else initial_callback + if callback is None: + raise RuntimeError("elicitation callback is unavailable") + return await callback(context, params) + + return elicitation_callback + case "logging_callback": + + async def logging_callback(params: LoggingMessageNotificationParams) -> None: + current_callback: Final = self._current_callbacks.get(name) + initial_callback: Final = self._initial_callbacks.get(name) + callback: Final = current_callback if current_callback is not None else initial_callback + if callback is None: + raise RuntimeError("logging callback is unavailable") + await callback(params) + + return logging_callback + assert_never(name) + + @property + def closed(self) -> bool: + return self._task.done() + + @property + def reusable(self) -> bool: + return not self._ended and not self._retiring and not self._task.done() + + def _record_stream_error(self, stream_error: asyncio.Future[Exception]) -> None: + self._stream_error = stream_error + + def _mark_ended(self) -> None: + self._ended = True + + async def _serve(self) -> None: + admission_stack: Final = AsyncExitStack() + + async def drain(session: ClientSession) -> None: + await admission_stack.aclose() + self._ready.set_result(None) + while True: + operation, future, callbacks = await self._queue.get() + if not future.done(): + self._active = future + if callbacks is not None: + self._current_callbacks = callbacks + async with self._admission() if self._admission is not None else nullcontext(): + if not future.done(): + (outcome,) = await asyncio.gather(operation(session), return_exceptions=True) + if _deliver_persistent_outcome(future, outcome, self._stream_error): + self._active = None + self._ended = True + return + self._active = None + if self._retiring and self._queue.empty(): + return + + try: + if self._admission is not None: + await admission_stack.enter_async_context(self._admission()) + (ended,) = await asyncio.gather( + self._client.run_with_session( + drain, + quiet_on_error=True, + on_stream_error=self._record_stream_error, + on_cleanup=self._mark_ended, + ), + return_exceptions=True, + ) + self._ended = True + except asyncio.CancelledError: + self._ended = True + self._fail_waiters(None) + raise + finally: + await admission_stack.aclose() + self._fail_waiters(ended if isinstance(ended, Exception) else None) + + def _fail_waiters(self, cause: Exception | None) -> None: + pending: Final = (self._ready, self._active, *(future for _, future, _ in self._drained())) + for future in pending: + if future is not None and not future.done(): + future.set_exception(UpstreamSessionClosedError() if cause is None else cause) + + def _drained(self) -> tuple[_PendingOperation, ...]: + return tuple(self._queue.get_nowait() for _ in range(self._queue.qsize())) + + async def run( + self, + operation: Callable[[ClientSession], Awaitable[TSessionResult]], + *, + quiet_on_error: bool = False, + callbacks: _SessionCallbackMap | None = None, + ) -> TSessionResult: + if self.closed: + await asyncio.shield(self._ready) + raise UpstreamSessionClosedError() + future: Final[asyncio.Future[object]] = asyncio.get_running_loop().create_future() + await self._queue.put((operation, future, callbacks)) + try: + _ = await asyncio.wait((future, self._task), return_when=asyncio.FIRST_COMPLETED) + except asyncio.CancelledError: + _ = future.cancel() + raise + if not future.done(): + raise UpstreamSessionClosedError() + return cast(TSessionResult, future.result()) # cast-ok: one queue serves operations of every result type + + def close(self) -> None: + _ = self._task.cancel() + + def retire(self) -> None: + if self._retiring: + return + self._retiring = True + if self.closed: + return + + async def retired_operation(_: ClientSession) -> None: + return None + + loop: Final = asyncio.get_running_loop() + future: Final[asyncio.Future[object]] = loop.create_future() + future.set_result(None) + try: + self._queue.put_nowait((retired_operation, future, None)) + except asyncio.QueueFull: + return + + async def wait_closed(self) -> None: + await asyncio.gather(self._task, return_exceptions=True) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 6eeb8f0582e..94a3564e5cb 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -38,7 +38,6 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.proxy._types import UserAPIKeyAuth from litellm.repositories.table_repositories import InvitationLinkRepository from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository @@ -52,6 +51,7 @@ from litellm.types.integrations.slack_alerting import ( VirtualKeyEvent, WebhookEvent, ) +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.proxy.model_deprecation import ( DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, DEPRECATION_IDLE_POLL_SECONDS, diff --git a/litellm/integrations/azure_storage/azure_storage.py b/litellm/integrations/azure_storage/azure_storage.py index 018ff758d5d..37b4d2d3937 100644 --- a/litellm/integrations/azure_storage/azure_storage.py +++ b/litellm/integrations/azure_storage/azure_storage.py @@ -2,7 +2,7 @@ import asyncio import os import time from collections.abc import Callable -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from functools import cache from typing import Final from urllib.parse import unquote @@ -28,7 +28,7 @@ from litellm.secret_managers.get_azure_ad_token_provider import ( from litellm.types.secret_managers.get_azure_ad_token_provider import ( AzureCredentialType, ) -from litellm.types.utils import StandardLoggingPayload +from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload AZURE_STORAGE_TOKEN_SCOPE: Final = "https://storage.azure.com/.default" _ADLS_SAFE_NAME: Final = str.maketrans("/", "_", "=") @@ -160,6 +160,24 @@ class AzureBlobStorageLogger(CustomBatchLogger): except Exception as e: verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e) + async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None: + now: Final = datetime.now(timezone.utc) + file_path: Final = f"audit_logs/{now.strftime('%Y-%m-%d')}/{now.strftime('%H-%M-%S')}_{audit_log['id']}.json" + await self._upload_json_to_file_path(file_path=file_path, json_payload=safe_dumps(audit_log)) + + async def _upload_json_to_file_path(self, file_path: str, json_payload: str) -> None: + payload_bytes: Final = json_payload.encode("utf-8") + if self.azure_storage_account_key: + await self._upload_bytes_with_account_key(file_path=file_path, content=payload_bytes) + return + + await self.set_valid_azure_ad_token() + async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) + base_url: Final = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{file_path}" + await self._create_file(async_client, base_url) + await self._append_data(async_client, base_url, json_payload) + await self._flush_data(async_client, base_url, len(payload_bytes)) + async def async_send_batch(self): """ Sends the in memory logs queue to Azure Blob Storage @@ -366,41 +384,22 @@ class AzureBlobStorageLogger(CustomBatchLogger): This is used when Azure Storage Account Key is set - Azure Storage Account Key does not work directly with Azure Rest API """ - # Create an async service client - - service_client: Final = await self.get_service_client() - # Get file system client - file_system_client: Final = service_client.get_file_system_client(file_system=self.azure_storage_file_system) - try: - # Create directory with today's date - from datetime import datetime - today: Final = datetime.now().strftime("%Y-%m-%d") - directory_client: Final = file_system_client.get_directory_client(today) - - # check if the directory exists - if not await directory_client.exists(): - await directory_client.create_directory() - verbose_logger.debug("Created directory: %s", today) - - # Create a file client file_name: Final = adls_safe_file_name(payload.get("id")) - file_client: Final = directory_client.get_file_client(file_name) - - # Create the file - await file_client.create_file() - - # Content to append - content: Final = safe_dumps(payload).encode("utf-8") - - # Append content to the file - await file_client.append_data(data=content, offset=0, length=len(content)) - - # Flush the content to finalize the file - await file_client.flush_data(position=len(content), offset=0) - + await self._upload_bytes_with_account_key( + file_path=f"{today}/{file_name}", content=safe_dumps(payload).encode("utf-8") + ) verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name) except Exception as e: verbose_logger.exception("Error occurred: %s", e) + + async def _upload_bytes_with_account_key(self, file_path: str, content: bytes) -> None: + service_client: Final = await self.get_service_client() + file_client: Final = service_client.get_file_system_client( + file_system=self.azure_storage_file_system + ).get_file_client(file_path) + await file_client.create_file() + await file_client.append_data(data=content, offset=0, length=len(content)) + await file_client.flush_data(position=len(content), offset=0) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index fab30deda93..45b20211ac5 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -44,7 +44,7 @@ from litellm.types.utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth dc: Final = DualCache() @@ -119,7 +119,7 @@ def is_guardrail_intervention(e: Exception) -> bool: def _user_api_key_auth_from_request(request_data: Mapping[str, object]) -> "UserAPIKeyAuth": - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth metadata: Final = request_data.get(get_metadata_variable_name_from_kwargs(request_data)) stamped: Final[Mapping[str, object]] = metadata if isinstance(metadata, dict) else {} diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 45fce665bf0..5ca9840185b 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -35,12 +35,12 @@ if TYPE_CHECKING: from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) - from litellm.proxy._types import UserAPIKeyAuth from litellm.types.mcp import ( MCPPostCallResponseObject, MCPPreCallRequestObject, MCPPreCallResponseObject, ) + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.router import PreRoutingHookResponse Span = _Span diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 7046b8d017a..6518cda5ae3 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -66,7 +66,7 @@ from ..additional_logging_utils import AdditionalLoggingUtils if TYPE_CHECKING: from fastapi import HTTPException - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth class _DatadogLoggingKwargs(TypedDict, total=False): diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 38f96f954f1..4b963ca3415 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -295,7 +295,9 @@ nothing here imports outside it: - [`config.py`](./model/config.py) — `OpenTelemetryV2Config`, a pydantic-settings model that reads `OTEL_*` / `LITELLM_OTEL_*` env vars, plus the feature gate. `capture_span_content` gates whether prompt/response bodies may be written as - span attributes; it defaults **off** (`no_content`). The Baggage allowlists are + span attributes; it defaults **off** (`no_content`). A team or key can override + the global mode per OTel v2 destination with `capture_message_content`; omitted + values follow the proxy's own callback for that backend, else the global mode. The Baggage allowlists are configurable, not hard-coded: set `LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS` / `LITELLM_OTEL_BAGGAGE_METADATA_KEYS` / `LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS` (comma-separated) as env vars, or diff --git a/litellm/integrations/otel/langfuse_logger.py b/litellm/integrations/otel/langfuse_logger.py index d029b153c52..c869474f532 100644 --- a/litellm/integrations/otel/langfuse_logger.py +++ b/litellm/integrations/otel/langfuse_logger.py @@ -13,7 +13,7 @@ from litellm.integrations.otel.model.trace_controls import caller_trace_controls from litellm.integrations.otel.plumbing.context import request_root_span if TYPE_CHECKING: - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.utils import ModelResponseStream @@ -51,6 +51,10 @@ class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2): response: "AsyncIterator[ModelResponseStream]", request_data: Mapping[str, object], ) -> "AsyncGenerator[ModelResponseStream, None]": + if not self._capture_span_content(): + async for chunk in response: + yield chunk + return relayed: Final[list[ModelResponseStream]] = [] # mutable-ok: relayed as they arrive, assembled at end of stream async for chunk in response: relayed.append(chunk) @@ -58,6 +62,8 @@ class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2): self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data)) def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None: + if not self._capture_span_content(): + return root: Final = request_root_span() if root is None or not root.is_recording(): return diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 498ebe655da..0844b043a6f 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -58,6 +58,7 @@ from litellm.integrations.otel.plumbing.context import ( is_recordable_span, mcp_message_transport_span, post_response_root, + request_destinations, request_root_http_route, request_root_span, resolve_internal_call_span_context, @@ -82,12 +83,13 @@ from litellm.integrations.otel.plumbing.providers import ( resolve_meter_provider, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache +from litellm.types.utils import captures_span_content if TYPE_CHECKING: from opentelemetry.metrics import MeterProvider from litellm.caching.dual_cache import DualCache - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.services import ServiceLoggerPayload from litellm.types.utils import ( CallTypesLiteral, @@ -254,6 +256,11 @@ class OpenTelemetryV2(CustomLogger): return None return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME), provider.resource) + def _capture_span_content(self) -> bool: + return self.config.capture_span_content or any( + captures_span_content(destination.capture_message_content) for destination in request_destinations() + ) + # ====================================================================== # # Proxy global registration # ====================================================================== # @@ -433,7 +440,7 @@ class OpenTelemetryV2(CustomLogger): return False payload: Final = cast("StandardLoggingPayload", raw_payload) data: Final = MCPToolCallSpanData.from_standard_logging_payload( - payload, capture_content=self.config.capture_span_content + payload, capture_content=self._capture_span_content() ) # A stray LLM carrier from a ``pre_call`` that mis-fired for this id would # otherwise linger until evicted; drop it so it's neither leaked nor closed @@ -477,7 +484,7 @@ class OpenTelemetryV2(CustomLogger): return False payload: Final = cast("StandardLoggingPayload", raw_payload) data: Final = MCPListToolsSpanData.from_standard_logging_payload( - payload, capture_content=self.config.capture_span_content + payload, capture_content=self._capture_span_content() ) if data.identity.call_id: self._release_carrier(self._open_llm_calls.pop(data.identity.call_id, None)) @@ -569,7 +576,7 @@ class OpenTelemetryV2(CustomLogger): return None data: Final = LLMCallSpanData.from_standard_logging_payload( payload, - capture_content=self.config.capture_span_content, + capture_content=self._capture_span_content(), time_to_first_chunk_seconds=call.time_to_first_chunk_seconds, request_route=request_root_http_route(), request_purpose=call.purpose, @@ -1076,6 +1083,6 @@ def build_otel_v2_logger( def _logger_class(config: OpenTelemetryV2Config) -> type[OpenTelemetryV2]: if "langfuse" not in config.mapper_names: return OpenTelemetryV2 - from litellm.integrations.otel.langfuse_logger import LangfuseContentOpenTelemetryV2, LangfuseOpenTelemetryV2 + from litellm.integrations.otel.langfuse_logger import LangfuseContentOpenTelemetryV2 - return LangfuseContentOpenTelemetryV2 if config.capture_span_content else LangfuseOpenTelemetryV2 + return LangfuseContentOpenTelemetryV2 diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index 3cf49ea2f11..55ef28bf470 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -16,19 +16,17 @@ from litellm.integrations.otel.model.baggage import ( ) from litellm.integrations.otel.model.spans import POSTGRESQL, db_system from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.utils import OtelSpanScope +from litellm.types.utils import ( + CAPTURE_MESSAGE_CONTENT_VALUES, + CaptureMessageContent, + OtelSpanScope, + captures_span_content, +) #: Master feature-flag env var. The logger is inert until this is truthy. OTEL_V2_ENV: Final = "LITELLM_OTEL_V2" -class CaptureMessageContent(str): - NO_CONTENT = "no_content" - SPAN_ONLY = "span_only" - EVENT_ONLY = "event_only" - SPAN_AND_EVENT = "span_and_event" - - class ExporterOwner(str, Enum): """The preset that contributed an exporter. Values match the callback names in ``presets.PRESET_BY_CALLBACK`` so per-request dynamic-credential routing @@ -297,9 +295,7 @@ class OpenTelemetryV2Config(BaseSettings): here are lower_snake_case; normalizing at the boundary keeps both spellings working and lets every downstream comparison stay exact. """ - if isinstance(value, str): - return value.lower() - return value + return _normalized_capture_message_content(value) @field_validator("langfuse_span_scope", mode="before") @classmethod @@ -371,22 +367,39 @@ class OpenTelemetryV2Config(BaseSettings): @property def capture_span_content(self) -> bool: - """Whether prompt/response content may be stamped as span attributes. - - Defaults off (``no_content``): an operator must opt in before message - bodies leave the process, so a user request can never force its prompt - or completion into the configured backend while capture is disabled. - """ - return self.capture_message_content in ( - CaptureMessageContent.SPAN_ONLY, - CaptureMessageContent.SPAN_AND_EVENT, - ) + """Whether the global setting permits prompt/response span attributes.""" + return captures_span_content(self.capture_message_content) @classmethod def from_env(cls) -> "OpenTelemetryV2Config": return cls() +def _normalized_capture_message_content(value: object) -> object: + if isinstance(value, str): + return value.lower() + return value + + +class _CaptureMessageContentSettings(BaseSettings): + model_config = SettingsConfigDict(extra="ignore", frozen=True) + + capture_message_content: str = Field( + default=CaptureMessageContent.NO_CONTENT, + validation_alias=AliasChoices("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"), + ) + + @field_validator("capture_message_content", mode="before") + @classmethod + def _normalize_capture_message_content(cls, value: object) -> object: + return _normalized_capture_message_content(value) + + +def capture_message_content_from_env() -> str: + """The global capture mode, read without building a settings model that rereads every other env var""" + return _CaptureMessageContentSettings().capture_message_content + + _EXCLUDED_SERVICES_INPUT: Final[TypeAdapter[str | tuple[object, ...]]] = TypeAdapter(str | tuple[object, ...]) @@ -395,6 +408,10 @@ def excluded_db_systems_from(value: object) -> frozenset[str]: return _normalize_excluded_services(excluded_service_names(value)) +def parse_capture_message_content(value: object) -> str | None: + return value if isinstance(value, str) and value in CAPTURE_MESSAGE_CONTENT_VALUES else None + + def excluded_service_names(value: object) -> frozenset[str]: """Read a YAML list or comma-separated string of service names, logging and dropping unusable input so a malformed value cannot stop the OTel logger from being built""" diff --git a/litellm/integrations/otel/model/destination.py b/litellm/integrations/otel/model/destination.py index 56b3123660c..b0c22dea292 100644 --- a/litellm/integrations/otel/model/destination.py +++ b/litellm/integrations/otel/model/destination.py @@ -11,7 +11,7 @@ from urllib.parse import quote from pydantic import ConfigDict, Field from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.utils import OtelSpanScope +from litellm.types.utils import OtelSpanScope, captures_span_content class OtelDestination(LiteLLMBaseModel): @@ -50,6 +50,17 @@ class OtelDestination(LiteLLMBaseModel): default=None, description="Same, for the requests with a failed span in their tree; ``None`` forwards every one.", ) + capture_message_content: str | None = Field( + default=None, + description=( + "An explicit mode overrides the global capture policy for this destination. " + "Omitted follows the global setting." + ), + ) + + def captures_content(self, default: str) -> bool: + setting: Final = self.capture_message_content + return captures_span_content(default if setting is None else setting) def header_string(self) -> str: """Render headers as the ``k=v,k2=v2`` form an ``ExporterSpec`` expects. @@ -64,9 +75,9 @@ class OtelDestination(LiteLLMBaseModel): def cache_key(self) -> tuple[str, tuple[tuple[str, str], ...], tuple[tuple[str, str], ...], str | None]: """Identity for processor reuse, so one destination means one exporter. - ``span_scope`` and the sampling rates are left out on purpose: they decide which - spans reach the processor, not how the processor exports them, so a full and an - ``llm_only`` view of the same account share one exporter. + ``span_scope``, the sampling rates and ``capture_message_content`` are left out on + purpose: they decide which spans reach the processor and what they carry, not how + the processor exports them, so two views of the same account share one exporter. """ return ( self.endpoint, diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 37a8f1951d9..4e7d4f231ed 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -5,7 +5,7 @@ import random import threading import time from collections import OrderedDict -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal @@ -40,8 +40,19 @@ from opentelemetry.util.types import Attributes, AttributeValue from litellm._logging import verbose_logger from litellm._version import version as litellm_version -from litellm.integrations.otel.mappers.langfuse import LANGFUSE_TRACE_NAME -from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.mappers.langfuse import ( + LANGFUSE_OBSERVATION_INPUT, + LANGFUSE_OBSERVATION_OUTPUT, + LANGFUSE_TRACE_NAME, +) +from litellm.integrations.otel.model.config import ( + CaptureMessageContent, + ExporterOwner, + ExporterSpec, + OpenTelemetryV2Config, + capture_message_content_from_env, + captures_span_content, +) from litellm.integrations.otel.model.semconv import ( DB, MCP, @@ -385,6 +396,26 @@ _CAPTURED_HEADER_PREFIXES: Final = ("http.request.header.", "http.response.heade # ``?key=`` query parameter. _URL_KEYS: Final = frozenset({"http.url", "http.target", "url.full"}) _URL_QUERY_KEY: Final = "url.query" +# Prompt and response content, in every vocabulary the mappers write it in once the +# global capture policy collects it: GenAI semconv (MCP tool-call arguments and results +# included), Langfuse, OpenInference's blobs and per-index messages, Langtrace, Weave. +_MESSAGE_CONTENT_KEYS: Final = frozenset( + { + GenAI.INPUT_MESSAGES, + GenAI.OUTPUT_MESSAGES, + GenAI.SYSTEM_INSTRUCTIONS, + GenAI.TOOL_CALL_ARGUMENTS, + GenAI.TOOL_CALL_RESULT, + LANGFUSE_OBSERVATION_INPUT, + LANGFUSE_OBSERVATION_OUTPUT, + "input.value", + "output.value", + "llm.prompts", + "llm.completions", + "weave.output", + } +) +_INDEXED_MESSAGE_PREFIXES: Final = ("llm.input_messages.", "llm.output_messages.") class _SpanView(ReadableSpan): @@ -415,6 +446,19 @@ class _SpanView(ReadableSpan): ) +def _is_message_content(key: str) -> bool: + return key in _MESSAGE_CONTENT_KEYS or key.startswith(_INDEXED_MESSAGE_PREFIXES) + + +def without_message_content(span: ReadableSpan) -> ReadableSpan: + """``span`` without its prompt and response content, everything else left as it was.""" + attributes: Final = span.attributes or _NO_ATTRIBUTES + if not any(_is_message_content(key) for key in attributes): + return span + kept: Final = MappingProxyType({key: value for key, value in attributes.items() if not _is_message_content(key)}) + return _SpanView(span, span.resource, kept, span.events, span.status, parent=span.parent) + + def _is_database_span(attributes: Mapping[str, AttributeValue]) -> bool: return any(key in attributes for key in _DB_SYSTEM_KEYS) @@ -553,7 +597,11 @@ def _tenant_resource(resource: Resource, defaults: Mapping[str, str], extra: Map return filled.merge(Resource(extra)) if extra else filled -def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> ReadableSpan: +def _for_destination( + span: ReadableSpan, + destination: "OtelDestination", + default_capture: str, +) -> ReadableSpan: """The view of ``span`` a tenant destination receives. A span the tenant's own call produced keeps its error text. Every other span is @@ -564,8 +612,10 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read exception it raised and names the operator's guardrail endpoint. Stack traces walk the operator's install and come off every span, as do the headers the operator captures on the server span, whose request side holds the caller's bearer token, - and the query string of the request URL, which can hold the same key. The span - itself stays, so the tenant still gets the whole trace tree. + and the query string of the request URL, which can hold the same key. A + destination whose effective setting disables span content loses the prompt and + response content too. The span itself stays, so the tenant still gets the whole + trace tree. """ extra: Final = destination.resource_attributes defaults: Final = destination.resource_defaults @@ -573,11 +623,12 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read database: Final = _is_database_span(attributes) owned: Final = _is_tenant_owned_span(attributes) unreachable: Final = _guardrail_unreachable(attributes) + redacted: Final = not destination.captures_content(default_capture) kept: Final = MappingProxyType( { key: _without_query(key, value) for key, value in attributes.items() - if _tenant_visible(key, database, owned, unreachable) + if _tenant_visible(key, database, owned, unreachable) and not (redacted and _is_message_content(key)) } ) recorded: Final = span.events @@ -612,6 +663,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): shutdown_drain_seconds: float = _SHUTDOWN_DRAIN_SECONDS, operator_sinks: 'Mapping[_SinkKey, "OtelSpanScope"]' = MappingProxyType({}), excluded_db_systems: frozenset[str] = frozenset(), + default_capture: str = CaptureMessageContent.NO_CONTENT, + default_capture_by_backend: Mapping[str, str] = MappingProxyType({}), pending_drains: int = _MAX_PENDING_DRAINS, drain_pool: _DrainPool | None = None, sampling_draw: Callable[[], float] | None = None, @@ -619,6 +672,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._operator_sinks: Final = operator_sinks self._draw: Final = sampling_draw if sampling_draw is not None else random.random self._excluded_db_systems: Final = excluded_db_systems + self._default_capture: Final = default_capture + self._default_capture_by_backend: Final = default_capture_by_backend self._drain_seconds: Final = shutdown_drain_seconds self._lock: Final = threading.Condition() self._closed = False # guarded by ``_lock``: an unlocked read races the teardown it gates @@ -661,12 +716,22 @@ class TenantFanOutSpanProcessor(SpanProcessor): for held_span, held_destination in self._settle(trace_id, failed): self._forward(held_span, held_destination) + def _default_capture_for(self, destination: "OtelDestination") -> str: + backend: Final = destination.callback_name + return ( + self._default_capture_by_backend.get(backend, self._default_capture) if backend else self._default_capture + ) + def _forward(self, span: ReadableSpan, destination: "OtelDestination") -> None: processor: Final = self._acquire(destination) if processor is None: return try: - processor.on_end(_scoped(_for_destination(span, destination), destination.span_scope)) + processor.on_end( + _scoped( + _for_destination(span, destination, self._default_capture_for(destination)), destination.span_scope + ) + ) except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span verbose_logger.debug("OTel V2 fan-out: forwarding to %s failed: %s", destination.endpoint, exc) finally: @@ -963,14 +1028,15 @@ class _OverriddenBackendFilter(SpanProcessor): Wrapping is the only place this works: ``SynchronousMultiSpanProcessor.on_end`` ignores return values, so a sibling processor can never veto the export. - Under ``additive`` mode nothing is suppressed, so the wrapper passes every span - straight through and the operator keeps its copy. + Under ``additive`` mode spans are not suppressed, though global content policy + still strips message content when disabled. ``scope`` narrows what the exporter receives independently of that: under ``llm_only`` the model-call spans go through as trace roots and the rest of the tree is held back, unless a destination of the request names ``sink``, the account this exporter writes to, with a wider scope: the fan-out then delivers the rest of - the tree there and the model call keeps its place in it. + the tree there and the model call keeps its place in it. A destination naming + ``sink`` sets this exporter's content policy; otherwise it follows global capture. """ def __init__( @@ -979,11 +1045,15 @@ class _OverriddenBackendFilter(SpanProcessor): owner: str | None, scope: "OtelSpanScope" = "full", sink: _SinkKey | None = None, + global_captures: bool = True, + filter_content: bool = True, ) -> None: self._inner: Final = inner self._owner: Final = owner self._scope: Final = scope self._sink: Final = sink + self._global_captures: Final = global_captures + self._filter_content: Final = filter_content def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: self._inner.on_start(span, parent_context) @@ -991,7 +1061,28 @@ class _OverriddenBackendFilter(SpanProcessor): def on_end(self, span: ReadableSpan) -> None: if self._owner in suppressed_backends() or not _in_scope(span, self._scope): return - self._inner.on_end(_scoped(span, self._account_scope())) + scoped: Final = _scoped(span, self._account_scope()) + self._inner.on_end( + without_message_content(scoped) if self._filter_content and not self._account_captures_content() else scoped + ) + + def _account_captures_content(self) -> bool: + """Whether a destination of this request that names ``sink`` captures content. + + The fan-out skips a destination this exporter already writes to, so this copy is + the one that account gets and has to honor the destination's content mode. + """ + explicit: Final = next( + ( + destination.capture_message_content + for destination in request_destinations() + if self._sink is not None + and destination.capture_message_content is not None + and _sink_key(destination.endpoint, destination.headers) == self._sink + ), + None, + ) + return captures_span_content(explicit) if explicit is not None else self._global_captures def _account_scope(self) -> "OtelSpanScope": if self._scope == "full" or self._sink is None: @@ -1010,6 +1101,25 @@ class _OverriddenBackendFilter(SpanProcessor): return self._inner.force_flush(timeout_millis) +class _MessageContentFilter(SpanProcessor): + """Strip prompt and response content from every span before ``inner`` exports it.""" + + def __init__(self, inner: SpanProcessor) -> None: + self._inner: Final = inner + + def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: + self._inner.on_start(span, parent_context) + + def on_end(self, span: ReadableSpan) -> None: + self._inner.on_end(without_message_content(span)) + + def shutdown(self) -> None: + self._inner.shutdown() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return self._inner.force_flush(timeout_millis) + + def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter: """Build a single exporter from the top-level config fields. @@ -1261,12 +1371,23 @@ def build_resource(config: OpenTelemetryV2Config) -> Resource: return Resource.create(attributes) +def _spec_processor( + spec: ExporterSpec, use_simple_processor: bool | None, strip_message_content: bool +) -> SpanProcessor: + exporting: Final = _processor_for( + _exporter_from_spec(spec), + (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), + ) + return _MessageContentFilter(exporting) if strip_message_content else exporting + + def build_tracer_provider( config: OpenTelemetryV2Config, exporter: SpanExporter | None = None, baggage_processor: SpanProcessor | None = None, use_simple_processor: bool | None = None, tenant_overrides: bool = False, + strip_message_content: bool = False, ) -> TracerProvider: """Build the shared :class:`TracerProvider`. @@ -1285,6 +1406,9 @@ def build_tracer_provider( ``config.langfuse_span_scope`` narrows the exporter owned by ``langfuse_otel`` alone; a collector or any other backend in the same config keeps the full tree. + ``strip_message_content`` removes prompt and response content before every + exporter, for a routed provider whose spans can carry content another + destination asked for. """ provider: Final = TracerProvider(resource=build_resource(config)) if baggage_processor is None: @@ -1300,17 +1424,23 @@ def build_tracer_provider( for spec in config.exporters: if spec.requires_headers and not spec.headers: continue - exp = _exporter_from_spec(spec) - processor = _processor_for( - exp, - (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), - ) + processor = _spec_processor(spec, use_simple_processor, strip_message_content) owner = spec.owner.value if tenant_overrides and spec.owner is not None else None scope = _operator_scope(config, spec) sink = _sink_key(spec.endpoint, parse_headers(spec.headers)) if _exports_to_the_wire(spec) else None provider.add_span_processor( - _OverriddenBackendFilter(processor, owner, scope, sink) - if owner is not None or scope != "full" + _OverriddenBackendFilter( + processor, + owner, + scope, + sink, + config.capture_span_content, + tenant_overrides, + ) + if owner is not None + or scope != "full" + or sink is not None + or (tenant_overrides and not config.capture_span_content) else processor ) return provider @@ -1340,10 +1470,29 @@ def attach_tenant_fan_out( TenantFanOutSpanProcessor( operator_sinks=operator_sink_scopes(*configs), excluded_db_systems=excluded_db_systems, + default_capture=_operator_capture_mode(configs), + default_capture_by_backend=MappingProxyType(dict(_owned_capture_modes(configs))), ) ) +def _operator_capture_mode(configs: Sequence[OpenTelemetryV2Config]) -> str: + """The mode of the operator's generic ``otel`` callback, so ``callback_settings.otel`` counts as the global default""" + return next( + (config.capture_message_content for config in configs if any(spec.owner is None for spec in config.exporters)), + capture_message_content_from_env(), + ) + + +def _owned_capture_modes( + configs: Sequence[OpenTelemetryV2Config], +) -> Iterator[tuple[str, str]]: + for config in configs: + for spec in config.exporters: + if spec.owner is not None: + yield spec.owner.value, config.capture_message_content + + def deliverable_destinations( destinations: Iterable["OtelDestination"], provider: trace.TracerProvider | None = None, diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index d7b1cadfc92..eb90b9035c5 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -282,7 +282,8 @@ class TenantTracerCache: self._providers.move_to_end(cache_key) return cached built: Final = build_tracer_provider( - self._routed_config(credential_headers, project_headers, endpoint, service_name) + self._routed_config(credential_headers, project_headers, endpoint, service_name), + strip_message_content=not self._config.capture_span_content, ) self._providers[cache_key] = built return built diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py index 91341a57634..c4273fbb337 100644 --- a/litellm/integrations/otel/presets/destinations.py +++ b/litellm/integrations/otel/presets/destinations.py @@ -187,6 +187,7 @@ def destination_for( callback_name: str, params: StandardCallbackDynamicParams, service_name: str | None = None, + capture_message_content: str | None = None, ) -> OtelDestination | None: """The destination ``params`` names for ``callback_name``, or ``None``. @@ -217,6 +218,7 @@ def destination_for( callback_name=callback_name, protocol=protocol, span_scope=_span_scope(callback_name, params), + capture_message_content=capture_message_content, success_sampling_rate=success_sampling_rate, error_sampling_rate=error_sampling_rate, ) diff --git a/litellm/integrations/otel/presets/newrelic.py b/litellm/integrations/otel/presets/newrelic.py index 771b3f643c8..9d8d04d8f0d 100644 --- a/litellm/integrations/otel/presets/newrelic.py +++ b/litellm/integrations/otel/presets/newrelic.py @@ -9,6 +9,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict from litellm._logging import verbose_logger from litellm.integrations.otel.model.config import ( + CaptureMessageContent, ExporterOwner, ExporterSpec, OpenTelemetryV2Config, @@ -66,7 +67,11 @@ def newrelic_preset( # New Relic ingests the OTLP GenAI semantic conventions natively. "mapper_names": ensure_mappers(base.mapper_names, "genai"), **( - {"capture_message_content": ("span_only" if settings.record_content else "no_content")} + { + "capture_message_content": ( + CaptureMessageContent.SPAN_ONLY if settings.record_content else CaptureMessageContent.NO_CONTENT + ) + } if settings.record_content is not None else {} ), diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 797c443e712..708a75aad99 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -53,7 +53,6 @@ from litellm.models.end_user import LiteLLM_EndUserTable from litellm.models.team import LiteLLM_TeamTable from litellm.models.user import LiteLLM_UserTable from litellm.models.verification_token import LiteLLM_DeletedVerificationToken -from litellm.proxy._types import UserAPIKeyAuth from litellm.repositories.base_repository import BaseRepository from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.organization_repository import OrganizationRepository @@ -67,6 +66,7 @@ from litellm.types.integrations.prometheus import ( sanitize_prometheus_label_value, validate_prometheus_deployment_and_latency_caller_identity, ) +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.proxy.carried_budget_state import ( KeyBudgetSnapshot, OrgBudgetSnapshot, diff --git a/litellm/integrations/rubrik.py b/litellm/integrations/rubrik.py index fe7264553df..57e8323cdc3 100644 --- a/litellm/integrations/rubrik.py +++ b/litellm/integrations/rubrik.py @@ -44,7 +44,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( Logging as LiteLLMLoggingObj, ) - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth _WEBHOOK_PATH_RESPONSE_MODERATION: Final = "/v1/after_completion/openai/v1" _WEBHOOK_PATH_PROMPT_MODERATION: Final = "/v1/before_prompt/openai/v1" diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index 046f510266a..962c7707409 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -631,13 +631,13 @@ async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool: """ try: from litellm.exceptions import BudgetExceededError - from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( get_team_object, team_max_budget_check, virtual_key_max_budget_check, ) from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth except ImportError: return False diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 0fb482cbbd9..c92e32aeb7f 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -77,10 +77,10 @@ if TYPE_CHECKING: from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) - from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.utils import ModelResponse from litellm.utils import CustomStreamWrapper diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 5d14a9761d5..a035f86ea56 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -122,8 +122,8 @@ class HealthCheckHelpers: 2. `user_api_key_auth`: This helps identify health check calls in the DB. We need this since the DB requires an API Key to track a log in the SpendLogs Table """ - from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth _metadata_variable_name: Final = "litellm_metadata" litellm_metadata: Final = HealthCheckHelpers._get_metadata_for_health_check_call() diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index d2ba17e169a..2efeead6bb2 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -6,6 +6,7 @@ from typing import Any, Final from litellm.types.utils import ( ARIZE_OTLP_PROTOCOLS, + CAPTURE_MESSAGE_CONTENT_VALUES, OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams, @@ -91,6 +92,13 @@ def validate_arize_otlp_protocol_value(value: str) -> None: raise ValueError(f"Invalid arize_otlp_protocol {value!r}: must be one of {sorted(ARIZE_OTLP_PROTOCOLS)}") +def validate_capture_message_content_value(value: str) -> None: + if value not in CAPTURE_MESSAGE_CONTENT_VALUES: + raise ValueError( + f"Invalid capture_message_content {value!r}: must be one of {sorted(CAPTURE_MESSAGE_CONTENT_VALUES)}" + ) + + # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict _supported_callback_params: Final[tuple[str, ...]] = ( "langfuse_public_key", diff --git a/litellm/litellm_core_utils/resource_ownership.py b/litellm/litellm_core_utils/resource_ownership.py new file mode 100644 index 00000000000..e0894b825a7 --- /dev/null +++ b/litellm/litellm_core_utils/resource_ownership.py @@ -0,0 +1,93 @@ +from typing import Final + +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.types.proxy.auth.user_roles import LitellmUserRoles + + +def is_proxy_admin(user_api_key_dict: UserAPIKeyAuth | None) -> bool: + if user_api_key_dict is None: + return False + + return ( + user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN + or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value + ) + + +def get_resource_owner_scopes( + user_api_key_dict: UserAPIKeyAuth | None, +) -> list[str]: + """ + Return ownership scopes that may access a user-created proxy resource. + + Raw user_id is included for rows created before scope prefixes existed. + Prefixes avoid collisions when falling back to team/org/key ownership + for keys that do not have a user_id. + + Identity-less callers (no user_id, team_id, org_id, api_key, or token) + return ``[]`` — they share no scope with any other caller, so access + checks against an existing owner always fail and creates that depend + on a primary scope must reject up front. Returning a shared sentinel + here would let any two identity-less callers see each other's data. + """ + if user_api_key_dict is None: + return [] + + scopes: Final[list[str]] = [] + + def _add(scope: str | None) -> None: + if scope and scope not in scopes: + scopes.append(scope) + + if user_api_key_dict.user_id: + _add(user_api_key_dict.user_id) + _add(f"user:{user_api_key_dict.user_id}") + if user_api_key_dict.team_id: + _add(f"team:{user_api_key_dict.team_id}") + if user_api_key_dict.org_id: + _add(f"org:{user_api_key_dict.org_id}") + if user_api_key_dict.api_key: + _add(f"key:{user_api_key_dict.api_key}") + if user_api_key_dict.token: + _add(f"key:{user_api_key_dict.token}") + + return scopes + + +def get_primary_resource_owner_scope( + user_api_key_dict: UserAPIKeyAuth | None, +) -> str | None: + """Return the canonical owner scope to stamp on newly-created rows. + + ``None`` for identity-less callers — callers that depend on a primary + scope to record ownership must surface that as a hard error rather + than fall back to a shared sentinel (which would collapse every + identity-less caller into the same logical owner). + """ + if user_api_key_dict is None: + return None + + if user_api_key_dict.user_id: + return user_api_key_dict.user_id + if user_api_key_dict.team_id: + return f"team:{user_api_key_dict.team_id}" + if user_api_key_dict.org_id: + return f"org:{user_api_key_dict.org_id}" + if user_api_key_dict.api_key: + return f"key:{user_api_key_dict.api_key}" + if user_api_key_dict.token: + return f"key:{user_api_key_dict.token}" + return None + + +def user_can_access_resource_owner( + owner: str | None, + user_api_key_dict: UserAPIKeyAuth | None, +) -> bool: + if user_api_key_dict is None: + return True + if is_proxy_admin(user_api_key_dict): + return True + if owner is None: + return False + return owner in get_resource_owner_scopes(user_api_key_dict) diff --git a/litellm/llms/a2a/chat/guardrail_translation/handler.py b/litellm/llms/a2a/chat/guardrail_translation/handler.py index 92dc49ea9c1..afd9ba84c96 100644 --- a/litellm/llms/a2a/chat/guardrail_translation/handler.py +++ b/litellm/llms/a2a/chat/guardrail_translation/handler.py @@ -26,7 +26,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth class _A2ATextPart(TypedDict, total=False): diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 7c1bee14b5b..717bafae5e7 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -76,10 +76,10 @@ if TYPE_CHECKING: ModifyResponseException, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth @dataclass(frozen=True, slots=True) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 454fb43b416..1bdb2b2db2c 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -33,8 +33,8 @@ from litellm.types.utils import ModelResponse from litellm.utils import get_model_info if TYPE_CHECKING: - from litellm.proxy._types import UserAPIKeyAuth from litellm.router import Router + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth # Anthropic-only keys already mapped by the translator; strip on extra_kwargs re-merge. ANTHROPIC_ONLY_REQUEST_KEYS: Final[frozenset[str]] = frozenset({"output_config", "safeguards"}) diff --git a/litellm/llms/anthropic/pass_through/context_management/dispatcher.py b/litellm/llms/anthropic/pass_through/context_management/dispatcher.py index 09fbaa89237..9b4e12ef2f9 100644 --- a/litellm/llms/anthropic/pass_through/context_management/dispatcher.py +++ b/litellm/llms/anthropic/pass_through/context_management/dispatcher.py @@ -13,8 +13,8 @@ from .editors import apply_clear_tool_uses_20250919, apply_compact_20260112 from .result import PolyfillResult if TYPE_CHECKING: - from litellm.proxy._types import UserAPIKeyAuth from litellm.router import Router + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth EditorResult: TypeAlias = "PolyfillResult | tuple[list[dict[str, object]], AppliedEdit | None]" diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/compact.py b/litellm/llms/anthropic/pass_through/context_management/editors/compact.py index 85a20ea4435..deb3d849ae9 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/compact.py +++ b/litellm/llms/anthropic/pass_through/context_management/editors/compact.py @@ -29,7 +29,6 @@ from litellm.types.llms.anthropic import ( if TYPE_CHECKING: from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper - from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.parallel_request_limiter_v3 import ( RateLimitDescriptor, RateLimitDescriptorRateLimitObject, @@ -41,6 +40,7 @@ if TYPE_CHECKING: AllAnthropicToolsValues, ) from litellm.types.llms.openai import ChatCompletionToolParam + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.utils import ModelResponse from ..constants import ( diff --git a/litellm/llms/base_llm/files/transformation.py b/litellm/llms/base_llm/files/transformation.py index cc36962da6b..083e396c0bd 100644 --- a/litellm/llms/base_llm/files/transformation.py +++ b/litellm/llms/base_llm/files/transformation.py @@ -22,9 +22,9 @@ from ..chat.transformation import BaseConfig if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer - from litellm.proxy._types import UserAPIKeyAuth as _UserAPIKeyAuth from litellm.router import Router as _Router from litellm.types.llms.openai import HttpxBinaryResponseContent + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth as _UserAPIKeyAuth LiteLLMLoggingObj = _LiteLLMLoggingObj Span = Any diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 70b8291d32c..88c4dbfdce6 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -11,8 +11,8 @@ if TYPE_CHECKING: ModifyResponseException, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import AllMessageValues + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth @dataclass(slots=True) diff --git a/litellm/llms/base_llm/managed_resources/base_managed_resource.py b/litellm/llms/base_llm/managed_resources/base_managed_resource.py index 81b3411ed56..bb4cf42fa1e 100644 --- a/litellm/llms/base_llm/managed_resources/base_managed_resource.py +++ b/litellm/llms/base_llm/managed_resources/base_managed_resource.py @@ -16,7 +16,7 @@ from litellm.llms.base_llm.managed_resources.isolation import ( can_access_resource, resolve_resource_owner_id, ) -from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.utils import SpecialEnums MANAGED_RESOURCES_TARGET: Final = "managed_resources" diff --git a/litellm/llms/base_llm/managed_resources/isolation.py b/litellm/llms/base_llm/managed_resources/isolation.py index 6a71e8e9223..547bed21247 100644 --- a/litellm/llms/base_llm/managed_resources/isolation.py +++ b/litellm/llms/base_llm/managed_resources/isolation.py @@ -12,10 +12,10 @@ are denied so an empty user_id can never select an unscoped query. from typing import Any, Final -from litellm.proxy._types import ( +from litellm.types.proxy.auth.user_api_key_auth import ( UserAPIKeyAuth, ) -from litellm.proxy._types import ( +from litellm.types.proxy.auth.user_api_key_auth import ( user_api_key_has_admin_view as _user_has_admin_view, ) @@ -29,7 +29,7 @@ def resolve_resource_owner_id( A key with neither a user_id nor a team_id would otherwise stamp ``created_by=None`` and be locked out of its own resources, so it owns them under its hashed token instead, using the ``key:`` scope prefix - already used by ``proxy/common_utils/resource_ownership.py``. ``None`` + already used by ``litellm_core_utils/resource_ownership.py``. ``None`` means the caller has no usable identity of its own and must fall back to team scoping, or be denied. """ diff --git a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py index eac8afd767c..912936a8ecd 100644 --- a/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py +++ b/litellm/llms/bedrock/passthrough/guardrail_translation/handler.py @@ -17,8 +17,8 @@ if TYPE_CHECKING: from litellm.llms.pass_through.guardrail_translation.handler import ( PassThroughEndpointHandler, ) - from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth _CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"}) _EVENT_STREAM_CONTENT_TYPE: Final = "vnd.amazon.eventstream" @@ -206,12 +206,14 @@ class _TextContainer(Protocol): _DeltaHolder = tuple[_GroupKey, _TextContainer, str] +_StreamGroupKey: TypeAlias = tuple[int | None, _GroupKey] + class _StreamFrame(TypedDict): """One raw event-stream frame plus the guardrailable texts it carries.""" raw: ReadOnly[bytes] - texts: ReadOnly[Sequence[tuple[_GroupKey, str]]] + texts: ReadOnly[Sequence[tuple[_StreamGroupKey, str]]] def _unpack_uint32(buffer: bytes) -> int: @@ -302,12 +304,14 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation): frames.append({"raw": frame_raw, "texts": []}) continue - texts: list[tuple[_GroupKey, str]] = [] + texts: list[tuple[_StreamGroupKey, str]] = [] if event_type == "contentBlockDelta": try: payload_dict: dict[str, object] = _json.loads(payload_bytes) + block_index = payload_dict.get("contentBlockIndex") + # Group per content block: a guardrail edit that changes length must not move text between blocks. texts = [ - (group_key, container[key]) + ((block_index if isinstance(block_index, int) else None, group_key), container[key]) for group_key, container, key in _collect_stream_delta_text_holders(payload_dict.get("delta")) ] except Exception as e: @@ -321,9 +325,9 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation): trailing_bytes: Final = body_bytes[offset:] - group_order: Final[list[_GroupKey]] = [] - group_members: Final[dict[_GroupKey, list[tuple[int, int]]]] = {} - group_texts: Final[dict[_GroupKey, list[str]]] = {} + group_order: Final[list[_StreamGroupKey]] = [] + group_members: Final[dict[_StreamGroupKey, list[tuple[int, int]]]] = {} + group_texts: Final[dict[_StreamGroupKey, list[str]]] = {} for frame_idx, frame in enumerate(frames): for local_idx, (group_key, text) in enumerate(frame["texts"]): if group_key not in group_members: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a8462c55332..134ab96ab56 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -213,7 +213,6 @@ if TYPE_CHECKING: FakeAnthropicMessagesStreamIterator, ) from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig - from litellm.proxy._types import UserAPIKeyAuth from litellm.types.google_genai.main import GenerateContentResponse from litellm.types.llms.openai_evals import ( CancelEvalResponse, @@ -225,6 +224,7 @@ if TYPE_CHECKING: Run, RunDeleteResponse, ) + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth LiteLLMLoggingObj = _LiteLLMLoggingObj else: diff --git a/litellm/llms/gemini/google_genai/guardrail_translation/handler.py b/litellm/llms/gemini/google_genai/guardrail_translation/handler.py index 0c1fe8171e8..44cdac11325 100644 --- a/litellm/llms/gemini/google_genai/guardrail_translation/handler.py +++ b/litellm/llms/gemini/google_genai/guardrail_translation/handler.py @@ -25,7 +25,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth _EMPTY_REQUEST_DATA: Final[Mapping[str, object]] = MappingProxyType({}) diff --git a/litellm/llms/litellm_proxy/skills/handler.py b/litellm/llms/litellm_proxy/skills/handler.py index 0dbe47791fb..77738ae99a2 100644 --- a/litellm/llms/litellm_proxy/skills/handler.py +++ b/litellm/llms/litellm_proxy/skills/handler.py @@ -11,19 +11,19 @@ from typing import Final from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache -from litellm.llms.litellm_proxy.skills.constants import ( - LITELLM_SKILL_ID_PREFIX, - MAX_SKILLS_PER_SEARCH, -) -from litellm.models.skills import LiteLLM_SkillsTable, NewSkillRequest -from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.common_utils.resource_ownership import ( +from litellm.litellm_core_utils.resource_ownership import ( get_primary_resource_owner_scope, get_resource_owner_scopes, is_proxy_admin, user_can_access_resource_owner, ) +from litellm.llms.litellm_proxy.skills.constants import ( + LITELLM_SKILL_ID_PREFIX, + MAX_SKILLS_PER_SEARCH, +) +from litellm.models.skills import LiteLLM_SkillsTable, NewSkillRequest from litellm.repositories.table_repositories import SkillsRepository +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth # Skills are looked up on every chat completion that has skills enabled # (`SkillsInjectionHook` calls ``fetch_skill_from_db``). 60s LRU/TTL cache diff --git a/litellm/llms/litellm_proxy/skills/skill_search.py b/litellm/llms/litellm_proxy/skills/skill_search.py index 98fc3bd2395..d8c1b6a368e 100644 --- a/litellm/llms/litellm_proxy/skills/skill_search.py +++ b/litellm/llms/litellm_proxy/skills/skill_search.py @@ -21,9 +21,9 @@ from litellm.types.utils import LlmProviders if TYPE_CHECKING: from litellm.models.skills import LiteLLM_SkillsTable - from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging from litellm.router import Router + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth DEFAULT_SKILL_SEARCH_TOP_K: Final = 5 MAX_SKILL_SEARCH_TOP_K: Final = 100 diff --git a/litellm/llms/litellm_proxy/skills/transformation.py b/litellm/llms/litellm_proxy/skills/transformation.py index 4cbd9f50301..73e26606497 100644 --- a/litellm/llms/litellm_proxy/skills/transformation.py +++ b/litellm/llms/litellm_proxy/skills/transformation.py @@ -22,7 +22,7 @@ from litellm.types.utils import LlmProviders if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.models.skills import LiteLLM_SkillsTable - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth class LiteLLMSkillsTransformationHandler: diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index d24075dabd3..a12f5f164e4 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -69,7 +69,7 @@ if TYPE_CHECKING: ModifyResponseException, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth class OpenAIChatCompletionsHandler(BaseTranslation): diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index a267b66bb22..61e0127f5b1 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -102,8 +102,8 @@ if TYPE_CHECKING: ModifyResponseException, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import ResponseInputParam + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth class _ToolCallShape(NamedTuple): diff --git a/litellm/llms/openai/videos/guardrail_translation/handler.py b/litellm/llms/openai/videos/guardrail_translation/handler.py index 7bdcc59ee7e..04511afa783 100644 --- a/litellm/llms/openai/videos/guardrail_translation/handler.py +++ b/litellm/llms/openai/videos/guardrail_translation/handler.py @@ -6,7 +6,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth class OpenAIVideoGenerationHandler(BaseTranslation): diff --git a/litellm/llms/pass_through/guardrail_translation/handler.py b/litellm/llms/pass_through/guardrail_translation/handler.py index 6e31569d421..a27ea70fbdb 100644 --- a/litellm/llms/pass_through/guardrail_translation/handler.py +++ b/litellm/llms/pass_through/guardrail_translation/handler.py @@ -18,8 +18,8 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth class PassThroughEndpointHandler(BaseTranslation): diff --git a/litellm/llms/scaledown/__init__.py b/litellm/llms/scaledown/__init__.py new file mode 100644 index 00000000000..d571093869e --- /dev/null +++ b/litellm/llms/scaledown/__init__.py @@ -0,0 +1,3 @@ +from .chat.transformation import ScaleDownChatConfig, ScaleDownError + +__all__ = ["ScaleDownChatConfig", "ScaleDownError"] diff --git a/litellm/llms/scaledown/chat/__init__.py b/litellm/llms/scaledown/chat/__init__.py new file mode 100644 index 00000000000..e6ac554eed1 --- /dev/null +++ b/litellm/llms/scaledown/chat/__init__.py @@ -0,0 +1,3 @@ +from .transformation import ScaleDownChatConfig, ScaleDownError + +__all__ = ["ScaleDownChatConfig", "ScaleDownError"] diff --git a/litellm/llms/scaledown/chat/transformation.py b/litellm/llms/scaledown/chat/transformation.py new file mode 100644 index 00000000000..44eb4785615 --- /dev/null +++ b/litellm/llms/scaledown/chat/transformation.py @@ -0,0 +1,648 @@ +import json +import time +import uuid +from collections.abc import Mapping, Sequence +from functools import reduce +from typing import TYPE_CHECKING, Final, NoReturn +from urllib.parse import unquote + +import httpx +from pydantic import JsonValue, TypeAdapter + +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices, Message, ModelResponse, Usage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +JSON_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]]) + +DEFAULT_API_BASE: Final = "https://api.scaledown.xyz" + +DOMAIN_MODELS: Final = frozenset({"classify", "extract", "summarize", "compress"}) + +DECISIONS_MODELS: Final = frozenset({"decisions"}) + +DECISIONS_UPSTREAM_MODEL: Final = "classify-1" + +NATIVE_PATHS: Final[Mapping[str, str]] = { + "classify": "/classify", + "extract": "/extract", + "summarize": "/summarization/abstractive", + "compress": "/compress/raw/", +} + +DECISION_QUESTION_TYPES: Final = frozenset({"choice", "noul", "score"}) + +EXTRA_BODY_KEYS: Final[Mapping[str, frozenset[str]]] = { + "extract": frozenset({"threshold", "top_n"}), + "summarize": frozenset(), + "compress": frozenset({"compression_rate"}), + "classify": frozenset({"labels"}), + "decisions": frozenset({"questions"}), +} + +# Set by the LiteLLM router and proxy on every call; handled by LiteLLM, never sent upstream. +ROUTER_PARAMS: Final = frozenset({"max_retries", "stream_options"}) + +SCORE_MIN_LEVELS: Final = 2 +SCORE_MAX_LEVELS: Final = 10 + +MAX_ENTITIES: Final = 1000 +MAX_REF_CHAIN: Final = 32 +MAX_SCHEMA_DEPTH: Final = 32 + + +class ScaleDownError(BaseLLMException): + pass + + +def _reject(message: str, status_code: int = 400) -> NoReturn: + """The single place this adapter raises its own errors.""" + raise ScaleDownError(status_code=status_code, message=message) + + +class ScaleDownChatConfig(BaseConfig): + @property + def custom_llm_provider(self) -> str | None: + return "scaledown" + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseConfig signature + model: str, + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + litellm_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, JsonValue]: # mutable-ok: BaseConfig signature + resolved_key: Final = api_key or get_secret_str("SCALEDOWN_API_KEY") + if resolved_key is None: + _reject("Missing ScaleDown API key. Set SCALEDOWN_API_KEY or pass api_key to the call.", 401) + if not api_key and api_base is not None and _root(api_base) != _trusted_root(): + _reject( + "A custom api_base needs its own api_key: the SCALEDOWN_API_KEY from the environment is only " + "sent to the default host or the host in SCALEDOWN_API_BASE." + ) + return {**headers, "x-api-key": resolved_key, "content-type": "application/json"} + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + litellm_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + stream: bool | None = None, + ) -> str: + base: Final = _root(api_base or _trusted_root()) + operation: Final = _operation(model) + if operation in DECISIONS_MODELS: + return f"{base}/v1/scaledown" + return f"{base}{NATIVE_PATHS[operation]}" + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseConfig signature + operation: Final = _operation(model) + if operation == "extract": + return ["response_format", "stream"] + if operation == "summarize": + return ["max_tokens", "max_completion_tokens", "stream"] + return ["stream"] + + def map_openai_params( + self, + non_default_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + model: str, + drop_params: bool, + ) -> dict[str, JsonValue]: # mutable-ok: BaseConfig signature + supported: Final = frozenset(self.get_supported_openai_params(model)) + unsupported: Final = sorted(key for key in non_default_params if key not in supported | ROUTER_PARAMS) + if unsupported and not drop_params: + _reject( + f"ScaleDown model '{model}' does not support {unsupported}. " + f"Supported: {sorted(supported)}. Set litellm.drop_params=True to ignore." + ) + mapped: Final = { + ("max_tokens" if key == "max_completion_tokens" else key): value + for key, value in non_default_params.items() + if key in supported + } + return {**optional_params, **mapped} + + def should_fake_stream( + self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None + ) -> bool: + return bool(stream) + + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: Mapping[str, object], + ) -> Mapping[str, JsonValue]: + # extra_body is merged after guardrails ran on the messages, so it may only add options. + # Anything that carries prompt text or picks the model is refused. + options: Final = JSON_OBJECT.validate_python(extra_body) + operation: Final = _operation(model) + allowed: Final = EXTRA_BODY_KEYS[operation] + unexpected: Final = sorted(set(options) - allowed) + if unexpected: + allowance: Final = f"may only set {sorted(allowed)}" if allowed else "is not accepted" + _reject( + f"extra_body for scaledown/{operation} {allowance}, got {unexpected}. " + "Text and instructions must be passed as messages." + ) + if operation == "compress": + return {"scaledown": {"rate": options["compression_rate"]}} if "compression_rate" in options else {} + if "questions" in options: + _validate_questions(options["questions"]) + if "labels" in options: + _validate_labels(options["labels"]) + return dict(options) + + def sign_request( + self, + headers: dict[str, str], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + request_data: dict[str, JsonValue], # mutable-ok: BaseConfig signature + api_base: str, + api_key: str | None = None, + model: str | None = None, + stream: bool | None = None, + fake_stream: bool | None = None, + ) -> tuple[dict[str, str], bytes | None]: # mutable-ok: BaseConfig signature + # Runs after extra_body is merged, so this is where a decisions body is known to be complete. + if model is not None and _operation(model) in DECISIONS_MODELS: + _require_complete_decisions(request_data) + if model is not None and _operation(model) == "classify": + _validate_labels(request_data.get("labels")) + return headers, None + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + litellm_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + headers: dict[str, str], # mutable-ok: BaseConfig signature + ) -> dict[str, JsonValue]: # mutable-ok: BaseConfig signature + checked_messages: Final = JSON_MESSAGES.validate_python(messages) + operation: Final = _operation(model) + _validate_text_input(operation, checked_messages, optional_params) + if operation == "classify": + return { + "text": _last_user_text(operation, checked_messages), + **({"labels": optional_params["labels"]} if "labels" in optional_params else {}), + } + if operation in DECISIONS_MODELS: + return dict(_decisions_request(checked_messages, optional_params)) + if operation == "extract": + return dict(_extract_request(checked_messages, optional_params)) + if operation == "summarize": + return dict(_summarize_request(checked_messages, optional_params)) + return dict(_compress_request(checked_messages, optional_params)) + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: dict[str, JsonValue], # mutable-ok: BaseConfig signature + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + litellm_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + encoding: "Tokenizer | None", + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ModelResponse: + try: + raw: Final = JSON_OBJECT.validate_json(raw_response.content) + except ValueError: + _reject( + f"ScaleDown returned a non-JSON response: {raw_response.text[:500]}", + raw_response.status_code if raw_response.is_error else 502, + ) + + operation: Final = _operation(model) + if operation in DECISIONS_MODELS: + return _decisions_response(operation, raw, model_response) + return _native_response(operation, raw, model_response, request_data.get("entities")) + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, # mutable-ok: BaseConfig signature + ) -> BaseLLMException: + return ScaleDownError(status_code=status_code, message=error_message, headers=headers) + + +def _root(api_base: str) -> str: + return api_base.rstrip("/").removesuffix("/v1") + + +def _trusted_root() -> str: + return _root(get_secret_str("SCALEDOWN_API_BASE") or DEFAULT_API_BASE) + + +def _operation(model: str) -> str: + operation: Final = model.split("/", 1)[1] if "/" in model else model + if operation not in DOMAIN_MODELS | DECISIONS_MODELS: + _reject(f"Unknown ScaleDown model '{model}'. Choose from {sorted(DOMAIN_MODELS | DECISIONS_MODELS)}.") + return operation + + +def _text_of(message: Mapping[str, JsonValue]) -> str | None: + content: Final = message.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + parts: Final = [ + str(part.get("text", "")) for part in content if isinstance(part, dict) and part.get("type") == "text" + ] + return "\n".join(parts) if parts else None + return None + + +def _validate_text_input( + operation: str, messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> None: + if any(key in optional_params for key in ("state", "document", "document_mime_type", "text")): + _reject("ScaleDown accepts text through messages only. Native state and document inputs are not supported.") + for message in messages: + if operation in {"classify", "decisions", "extract"} and message.get("role") in {"system", "developer"}: + _reject( + f"ScaleDown {operation} does not support system messages. Put instructions in the field definitions." + ) + if _has_non_text_content(message): + _reject("ScaleDown currently supports text only. Image and document inputs are not supported.") + + +def _has_non_text_content(message: Mapping[str, JsonValue]) -> bool: + content: Final = message.get("content") + return isinstance(content, list) and any( + not isinstance(part, dict) or part.get("type") != "text" for part in content + ) + + +def _last_user_text(operation: str, messages: Sequence[Mapping[str, JsonValue]]) -> str: + message: Final = next((message for message in reversed(messages) if message.get("role") == "user"), None) + text: Final = _text_of(message) if message is not None else None + if not text: + _reject(f"ScaleDown {operation} requires text in the last user message.") + return text + + +def _validate_labels(labels: JsonValue) -> None: + if not isinstance(labels, list) or not labels: + _reject("ScaleDown classify requires a non-empty 'labels' list of objects with 'name' and 'rubric' strings.") + for label in labels: + if not isinstance(label, Mapping) or not all( + isinstance(label.get(key), str) and label[key] for key in ("name", "rubric") + ): + _reject("Each classify label needs non-empty 'name' and 'rubric' strings.") + + +def _child(node: JsonValue, key: str) -> JsonValue: + return node.get(key.replace("~1", "/").replace("~0", "~")) if isinstance(node, Mapping) else None + + +def _extend_ref_path(path: tuple[str, ...], ref: str) -> tuple[str, ...]: + return (*path, ref) + + +def _nullable_branch(schema: Mapping[str, JsonValue]) -> Mapping[str, JsonValue] | None: + if "allOf" in schema: + _reject("ScaleDown extraction does not support allOf. Supply the object's properties directly.") + variants: Final = schema.get("anyOf", schema.get("oneOf")) + if variants is None: + return None + if any(key in schema for key in ("properties", "items", "$ref", "type")) or ( + "anyOf" in schema and "oneOf" in schema + ): + _reject("ScaleDown extraction cannot combine nullable branches with sibling schema definitions.") + if not isinstance(variants, list) or len(variants) != 2: + _reject("ScaleDown extraction supports anyOf/oneOf only for one schema plus null.") + branches: Final = tuple(branch for branch in variants if isinstance(branch, dict)) + non_null: Final = tuple(branch for branch in branches if branch.get("type") != "null") + if len(branches) != 2 or len(non_null) != 1: + _reject("ScaleDown extraction supports anyOf/oneOf only for one schema plus null.") + return { + **non_null[0], + **{key: value for key, value in schema.items() if key not in {"anyOf", "oneOf"}}, + } + + +def _deref( + prop: Mapping[str, JsonValue], root: Mapping[str, JsonValue], path: tuple[str, ...] +) -> tuple[Mapping[str, JsonValue], tuple[str, ...], bool]: + """Follow nullable branches and local `$ref`s (`#/$defs/X`, `#/definitions/X`). + + Returns the schema, the refs followed so far, and whether a ref repeated (a cycle). + The walk is a bounded loop, so a long chain of distinct refs cannot exhaust the stack. + """ + current: Mapping[str, JsonValue] = prop # rebind-ok: bounded walk along a ref chain + followed: tuple[str, ...] = path # rebind-ok: bounded walk along a ref chain + for _ in range(MAX_REF_CHAIN): + nullable: Mapping[str, JsonValue] | None = _nullable_branch(current) + ref: JsonValue = current.get("$ref") + if nullable is not None: + current = nullable + continue + if "$ref" not in current: + return current, followed, False + if not isinstance(ref, str) or not ref.startswith("#"): + _reject("ScaleDown extraction supports only local JSON Pointer $refs.") + pointer: str = unquote(ref[1:]) + if not pointer.startswith("/"): + _reject("ScaleDown extraction supports only local JSON Pointer $refs beginning with #/.") + if any(key in current for key in ("properties", "items", "type")): + _reject("ScaleDown extraction cannot combine $ref with sibling schema definitions.") + if ref in followed: + return current, followed, True + target: JsonValue = reduce(_child, pointer[1:].split("/"), dict(root)) + if not isinstance(target, Mapping): + _reject(f"ScaleDown extraction cannot resolve $ref '{ref}' to a schema object.") + current, followed = target, _extend_ref_path(followed, ref) + _reject(f"The response_format schema follows more than {MAX_REF_CHAIN} chained $refs or nullable branches.") + + +def _build_properties( + properties: Mapping[str, JsonValue], + root: Mapping[str, JsonValue], + path: tuple[str, ...], + budget: int, + depth: int = 0, +) -> tuple[dict[str, JsonValue], int]: # mutable-ok: serialized JSON entity map + """Turn JSON-schema properties into the /extract entity map, spending one budget unit per entity. + + Each property becomes an entity whose value is its description (or its name when there is + none). Nested objects stay nested, arrays of objects become a one-element list holding the + item's entity map, local `$ref`s are followed, and a ref that points back into itself is cut + off at that point instead of expanded. + """ + if depth > MAX_SCHEMA_DEPTH: + _reject(f"The response_format schema nests deeper than {MAX_SCHEMA_DEPTH} levels.") + built: Final[dict[str, JsonValue]] = {} # mutable-ok: filled once while threading the budget through + remaining = budget # rebind-ok: the budget is threaded through the loop + for name, prop in properties.items(): + if remaining <= 0: + _reject( + f"The response_format schema expands to more than {MAX_ENTITIES} entities; " + "remove recursive or heavily repeated $refs." + ) + built[name], remaining = _build_entity(name, prop, root, path, remaining - 1, depth) + return built, remaining + + +def _build_entity( + name: str, prop: JsonValue, root: Mapping[str, JsonValue], path: tuple[str, ...], budget: int, depth: int +) -> tuple[JsonValue, int]: + schema, followed, cyclic = _deref(prop if isinstance(prop, Mapping) else {}, root, path) + leaf: Final = schema.get("description") or name + nested: Final = schema.get("properties") + if cyclic: + return leaf, budget + if isinstance(nested, Mapping): + return _build_properties(nested, root, followed, budget, depth + 1) + raw_items: Final = schema.get("items") + schema_type: Final = schema.get("type") + is_array: Final = schema_type == "array" or (isinstance(schema_type, list) and "array" in schema_type) + if is_array and isinstance(raw_items, Mapping): + items, item_path, item_cyclic = _deref(raw_items, root, followed) + item_properties: Final = items.get("properties") + if not item_cyclic and isinstance(item_properties, Mapping): + built, remaining = _build_properties(item_properties, root, item_path, budget, depth + 1) + return [built], remaining + return leaf, budget + + +def _extract_request( + messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + response_format: Final = optional_params.get("response_format") + json_schema: Final = response_format.get("json_schema") if isinstance(response_format, Mapping) else None + if isinstance(json_schema, Mapping) and json_schema.get("strict") is True: + _reject( + "ScaleDown extract does not support strict JSON schemas. Use strict=false and validate the returned fields." + ) + schema: Final = json_schema.get("schema") if isinstance(json_schema, Mapping) else None + properties: Final = _deref(schema, schema, ())[0].get("properties") if isinstance(schema, Mapping) else None + if not isinstance(schema, dict) or not isinstance(properties, dict) or not properties: + _reject( + "ScaleDown extract needs response_format={'type': 'json_schema', ...} with a non-empty " + "'properties' map; each property becomes an entity and its description the extraction hint." + ) + return { + "text": _last_user_text("extract", messages), + "entities": _build_properties(properties, schema, (), MAX_ENTITIES)[0], + **{key: optional_params[key] for key in ("threshold", "top_n") if key in optional_params}, + } + + +def _summarize_request( + messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + instructions: Final = [ + text for message in messages if message.get("role") in {"system", "developer"} and (text := _text_of(message)) + ] + return { + "text": _last_user_text("summarize", messages), + **({"instructions": "\n".join(instructions)} if instructions else {}), + **({"max_tokens": optional_params["max_tokens"]} if "max_tokens" in optional_params else {}), + } + + +def _compress_request( + messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + prompt: Final = _last_user_text("compress", messages) + last_user_index: Final = max(i for i, m in enumerate(messages) if m.get("role") == "user" and _text_of(m)) + context: Final = "\n\n".join(text for i, m in enumerate(messages) if i != last_user_index and (text := _text_of(m))) + return { + "context": context, + "prompt": prompt, + "scaledown": {"rate": optional_params.get("compression_rate", "auto")}, + } + + +def _decisions_request( + messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + questions: Final = optional_params.get("questions") + if questions is not None: + _validate_questions(questions) + return { + "model": DECISIONS_UPSTREAM_MODEL, + "state": {"text": _last_user_text("decisions", messages)}, + **({"questions": questions} if questions is not None else {}), + } + + +def _require_complete_decisions(body: Mapping[str, JsonValue]) -> None: + if not body.get("questions"): + _reject( + "ScaleDown decisions requires a non-empty 'questions' map, passed as an extra parameter, " + 'e.g. {"questions": {"category": {"type": "choice", "criteria": {"billing": "..."}}}}. ' + "Question types are 'choice', 'noul', and 'score'." + ) + + +def _validate_questions(questions: JsonValue) -> None: + if not isinstance(questions, Mapping): + _reject(f"'questions' must be a map of name to question, got {type(questions).__name__}.") + for name, question in questions.items(): + if not isinstance(question, Mapping): + _reject(f"Question '{name}' must be an object.") + question_type = question.get("type") + if not isinstance(question_type, str) or question_type not in DECISION_QUESTION_TYPES: + _reject(f"Question '{name}' has type {question_type!r}; expected one of {sorted(DECISION_QUESTION_TYPES)}.") + criteria = question.get("criteria") + if question_type == "choice" and (not isinstance(criteria, Mapping) or not criteria): + _reject(f"Choice question '{name}' needs a non-empty 'criteria' map of option key to description.") + if question_type == "score" and ( + not isinstance(criteria, list) or not SCORE_MIN_LEVELS <= len(criteria) <= SCORE_MAX_LEVELS + ): + _reject( + f"Score question '{name}' needs 'criteria' as an ordered list of " + f"{SCORE_MIN_LEVELS} to {SCORE_MAX_LEVELS} level descriptions, lowest to highest." + ) + + +def _unwrap(value: JsonValue) -> JsonValue: + wrapper_keys: Final = {"_value", "_span_anchor"} + if isinstance(value, Mapping) and "_value" in value and set(value) <= wrapper_keys: + return value["_value"] + return value + + +def _clean_extraction(value: JsonValue, requested: JsonValue) -> JsonValue: + """Reduce an extraction result to the fields the caller's schema asked for. + + The request's entity map says which keys are real fields, so provider-added keys such as + `_span_anchor` and `{"_value": ..., "_span_anchor": ...}` wrappers are dropped without + touching a requested field that happens to share one of those names. The untouched payload + stays on `_hidden_params["scaledown_response"]`. + """ + if isinstance(requested, Mapping) and isinstance(value, Mapping): + return {key: _clean_extraction(value[key], requested[key]) for key in requested if key in value} + if isinstance(requested, list) and requested and isinstance(value, list): + return [_clean_extraction(item, requested[0]) for item in value] + return _unwrap(value) + + +def _extracted_fields(raw: Mapping[str, JsonValue], requested: JsonValue) -> JsonValue: + entities: Final = raw.get("entities") + matches: Final = ( + tuple(entity for entity in entities if isinstance(entity, Mapping)) if isinstance(entities, list) else () + ) + scalar_names: Final = ( + tuple(name for name, hint in requested.items() if isinstance(hint, str)) + if isinstance(requested, Mapping) + else () + ) + scalars: Final = { + name: next(entity["text"] for entity in matches if entity.get("type") == name and "text" in entity) + for name in scalar_names + if any(entity.get("type") == name and "text" in entity for entity in matches) + } + structured: Final = raw.get("structured_result") + return _clean_extraction({**scalars, **(structured if isinstance(structured, Mapping) else {})}, requested) + + +def _native_response( + operation: str, raw: JsonValue, model_response: ModelResponse, requested: JsonValue = None +) -> ModelResponse: + """Wrap a native /extract, /summarization/abstractive or /compress/raw/ payload. + + Summarize and compress return the upstream payload as JSON on choices[0].message.content. + Extract returns the fields requested by response_format without enforcing schema types. + The native APIs report input tokens only; they return no output token + count, so completion_tokens stays 0 because it is unmeasured, not because nothing was generated. + """ + if not isinstance(raw, Mapping) or not raw: + _reject( + f"ScaleDown '{operation}' returned an empty or malformed response: {json.dumps(raw)[:500]}", + 500, + ) + + results: Final = raw.get("results") + input_tokens: Final = raw.get("input_tokens") or ( + (results.get("original_prompt_tokens") if isinstance(results, Mapping) else None) + or raw.get("original_prompt_tokens") + if operation == "compress" + else None + ) + content: Final = _extracted_fields(raw, requested) if operation == "extract" else raw + + model_response.id = f"chatcmpl-{uuid.uuid4().hex}" # rebind-ok: LiteLLM fills the response object it passes in + model_response.created = int(time.time()) # rebind-ok: LiteLLM fills the response object it passes in + model_response.model = f"scaledown/{operation}" # rebind-ok: LiteLLM fills the response object it passes in + model_response.object = "chat.completion" # rebind-ok: LiteLLM fills the response object it passes in + model_response.choices = [ # rebind-ok: LiteLLM fills the response object it passes in + Choices( + index=0, + message=Message(role="assistant", content=json.dumps(content, separators=(",", ":"))), + finish_reason="stop", + ) + ] + _set_usage(model_response, {"prompt_tokens": input_tokens or 0}) + model_response.hidden_params["scaledown_response"] = raw # pyright: ignore[reportUnknownMemberType] # Inherited hidden_params has untyped values. # rebind-ok: supplied response + return model_response + + +def _decisions_response(operation: str, raw: JsonValue, model_response: ModelResponse) -> ModelResponse: + answers: Final = raw.get("answers") if isinstance(raw, Mapping) else None + if answers is None: + _reject(f"ScaleDown decisions response contained no answers: {json.dumps(raw)[:500]}", 500) + + model_response.id = f"chatcmpl-{uuid.uuid4().hex}" # rebind-ok: LiteLLM fills the response object it passes in + model_response.created = int(time.time()) # rebind-ok: LiteLLM fills the response object it passes in + model_response.model = f"scaledown/{operation}" # rebind-ok: LiteLLM fills the response object it passes in + model_response.object = "chat.completion" # rebind-ok: LiteLLM fills the response object it passes in + model_response.choices = [ # rebind-ok: LiteLLM fills the response object it passes in + Choices( + index=0, + message=Message(role="assistant", content=json.dumps(answers, separators=(",", ":"))), + finish_reason="stop", + ) + ] + + usage: Final = raw.get("usage") if isinstance(raw, dict) else None + counts: Final = usage if isinstance(usage, dict) else {} + _set_usage( + model_response, + { + "prompt_tokens": counts.get("input_tokens", 0), + "completion_tokens": counts.get("output_tokens", 0), + }, + ) + model_response.hidden_params["scaledown_response"] = raw # pyright: ignore[reportUnknownMemberType] # Inherited hidden_params has untyped values. # rebind-ok: supplied response + return model_response + + +def _set_usage(model_response: ModelResponse, usage: Mapping[str, JsonValue]) -> None: + prompt_tokens: Final = _token_count(usage.get("prompt_tokens")) + completion_tokens: Final = _token_count(usage.get("completion_tokens")) + total_tokens: Final = usage.get("total_tokens") + model_response.usage = Usage( # pyright: ignore[reportAttributeAccessIssue] # ModelResponse stores usage as a dynamic Pydantic field. # rebind-ok: fills the supplied response + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=_token_count(total_tokens) if total_tokens is not None else prompt_tokens + completion_tokens, + ) + + +def _token_count(value: JsonValue) -> int: + if value is None: + return 0 + if isinstance(value, int) and value >= 0: + return value + _reject("ScaleDown returned an invalid token count.", 500) diff --git a/litellm/main.py b/litellm/main.py index 88c1257ee2c..c2c6d5affe8 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1938,6 +1938,39 @@ def _complete_ragflow(ctx: CompletionDispatchContext) -> _CompletionDispatchResu return response +def _complete_scaledown(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: + try: + return base_llm_http_handler.completion( + model=ctx.model, + messages=ctx.messages, + headers=ctx.headers, + model_response=ctx.model_response, + api_key=ctx.api_key, + api_base=ctx.api_base, + acompletion=ctx.acompletion, + logging_obj=ctx.logging, + optional_params=ctx.optional_params, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + timeout=cast( # cast-ok: CompletionTimeout.resolve normalizes this value + float | httpx.Timeout, ctx.timeout + ), + client=_dispatch_client_http(ctx), + custom_llm_provider=ctx.custom_llm_provider, + encoding=_get_encoding(), + stream=ctx.stream, + provider_config=ctx.provider_config, + ) + except Exception as exc: + ctx.logging.post_call( + input=ctx.messages, + api_key=ctx.api_key, + original_response=str(exc), + additional_args={"headers": ctx.headers}, + ) + raise + + def _complete_xai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: acompletion: Final = ctx.acompletion api_base: Final = ctx.api_base @@ -5859,6 +5892,8 @@ def completion( elif custom_llm_provider == "ragflow": ## COMPLETION CALL - RAGFlow uses HTTP handler to support custom URL paths response = _complete_ragflow(_dispatch_ctx) + elif custom_llm_provider == "scaledown": + response = _complete_scaledown(_dispatch_ctx) # rebind-ok: shared provider return elif custom_llm_provider == "xai": ## COMPLETION CALL response = _complete_xai(_dispatch_ctx) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cb910ae12ea..bf7f1db7a31 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -66876,7 +66876,7 @@ "supports_audio_input": true }, "azure/gpt-realtime-translate": { - "deprecation_date": "2027-05-06", + "deprecation_date": "2027-11-09", "input_cost_per_second": 0.000566666666667, "litellm_provider": "azure", "max_input_tokens": 32000, @@ -71842,5 +71842,51 @@ "image" ], "supports_embedding_image_input": true + }, + "scaledown/extract": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/summarize": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": true, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/compress": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": true, + "supports_function_calling": false, + "supports_vision": false + }, + "scaledown/classify": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/decisions": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false } } diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index d9a6feb625d..a0e75b1f02d 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2064,6 +2064,24 @@ "search": true } }, + "scaledown": { + "display_name": "ScaleDown (`scaledown`)", + "url": "https://docs.litellm.ai/docs/providers/scaledown", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "sambanova": { "display_name": "Sambanova (`sambanova`)", "url": "https://docs.litellm.ai/docs/providers/sambanova", diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index bb2fe2d59b4..2c4c942ebeb 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -95,9 +95,7 @@ def _normalize_caller_admission_credential(value: str) -> str: def _admission_request(scope: Scope) -> Request: - """Request whose ``body()`` serves the routing layer's lazily peeked JSON-RPC - bytes (``b"{}"`` when no peek callable was stashed) instead of the ASGI - receive channel.""" + """Request whose ``body()`` serves the peeked JSON-RPC method (``b"{}"`` when nothing is stashed).""" request: Final = Request(scope=scope) peeked_body: Final[Callable[[], Awaitable[bytes]] | None] = scope.get(MCP_PEEKED_BODY_SCOPE_KEY) diff --git a/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py b/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py index 17bb84e066a..31d1ad8ccc2 100644 --- a/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py +++ b/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py @@ -27,6 +27,10 @@ class ElicitationCallback(Protocol): async def __call__(self, context: object, params: ElicitRequestParams) -> ElicitResult | ErrorData: ... +def has_active_mcp_request_context() -> bool: + return get_active_mcp_request_ctx() is not None + + def create_sampling_callback( user_api_key_auth: UserAPIKeyAuth | None = None, raw_headers: Mapping[str, str] | None = None, diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index e492af133ff..abfed7dd33c 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -24,7 +24,7 @@ from collections.abc import ( MutableMapping, Sequence, ) -from contextlib import ExitStack, asynccontextmanager +from contextlib import AbstractAsyncContextManager, ExitStack, asynccontextmanager, nullcontext from dataclasses import dataclass, replace from functools import lru_cache from itertools import chain, groupby @@ -69,9 +69,16 @@ from litellm.constants import ( MCP_HEALTH_CHECK_TIMEOUT, MCP_METADATA_TIMEOUT, MCP_TOOL_LISTING_TIMEOUT, + MCP_UPSTREAM_SESSION_SHUTDOWN_TIMEOUT_SECONDS, ) from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException -from litellm.experimental_mcp_client.client import MCPClient, strip_auth_scheme, to_basic_credentials +from litellm.experimental_mcp_client.client import ( + MCPClient, + PersistentMCPSession, + UpstreamSessionClosedError, + strip_auth_scheme, + to_basic_credentials, +) from litellm.integrations.custom_guardrail import ( _sync_guardrail_info_to_logging_obj, # pyright: ignore[reportPrivateUsage] - the same bridge @log_guardrail_information uses; reimplementing it here would fork the metadata-key logic ) @@ -98,6 +105,7 @@ from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( upstream_auth_challenge, ) from litellm.proxy._experimental.mcp_server.interactions import bind_target +from litellm.proxy._experimental.mcp_server.legacy_callbacks import has_active_mcp_request_context from litellm.proxy._experimental.mcp_server.mcp_debug import describe_upstream_http_failure, record_auth_resolution from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( MCPPerUserTokenCache, @@ -343,6 +351,12 @@ _ToolParamMap: TypeAlias = dict[str, list[str]] _EnvVarList: TypeAlias = list[dict[str, object]] _InMemoryCacheDict: TypeAlias = dict[str, object] _ToolArguments: TypeAlias = dict[str, object] +_UpstreamSessionKey: TypeAlias = tuple[ + str, + str, + str, + tuple[MCPTransport, MCPUpstreamProtocol, float, bool, bool, bool], +] @dataclass(frozen=True, slots=True) @@ -1927,6 +1941,9 @@ class MCPServerManager: self.catalog = CatalogSnapshots(self) self.registry: dict[str, MCPServer] = {} + self._upstream_sessions: dict[_UpstreamSessionKey, PersistentMCPSession] = {} # mutable-ok: session registry + self._retiring_upstream_sessions: list[tuple[str, PersistentMCPSession]] = [] # mutable-ok: session registry + self._live_gateway_sessions: frozenset[str] = frozenset() self._openapi_health_probes: Callable[[str], _OpenAPIHealthProbe] = lru_cache(maxsize=128)(_OpenAPIHealthProbe) self.config_mcp_servers: dict[str, MCPServer] = {} """ @@ -6123,6 +6140,108 @@ class MCPServerManager: async with semaphore: yield + def _outbound_limit( + self, + mcp_server: MCPServer, + persistent_session: PersistentMCPSession | None, + ) -> AbstractAsyncContextManager[None]: + if persistent_session is not None: + return nullcontext() + return self._limit_outbound_concurrency(mcp_server) + + async def _upstream_session_for( + self, + client: MCPClient, + mcp_server: MCPServer, + raw_headers: Mapping[str, str] | None, + ) -> PersistentMCPSession | None: + gateway_session_id: Final = next( + ( + value + for key, value in (raw_headers.items() if raw_headers else ()) + if key.lower() == "mcp-session-id" and value + ), + None, + ) + if ( + not has_active_mcp_request_context() + or gateway_session_id is None + or gateway_session_id not in self._live_gateway_sessions + or mcp_server.transport == MCPTransport.stdio + ): + return None + key: Final[_UpstreamSessionKey] = ( + gateway_session_id, + mcp_server.server_id, + await client.discovery_auth_fingerprint(), + client.session_settings(), + ) + if gateway_session_id not in self._live_gateway_sessions: + return None + existing: Final = self._upstream_sessions.get(key) + if existing is not None and existing.reusable: + return existing + opened: Final = client.open_persistent_session( + lambda: self._limit_outbound_concurrency(self.get_mcp_server_by_id(mcp_server.server_id) or mcp_server) + ) + self._retiring_upstream_sessions = [ + (gateway_session_id, session) + for gateway_session_id, session in self._retiring_upstream_sessions + if not session.closed + ] + previous_keys: Final = tuple( + registered_key for registered_key in self._upstream_sessions if registered_key[:2] == key[:2] + ) + previous_sessions: Final = tuple( + (key[0], self._upstream_sessions.pop(previous_key)) for previous_key in previous_keys + ) + for retired_gateway_session_id, previous_session in previous_sessions: + previous_session.retire() + self._retiring_upstream_sessions.append((retired_gateway_session_id, previous_session)) + self._upstream_sessions[key] = opened + return opened + + def track_gateway_session(self, gateway_session_id: str) -> None: + self._live_gateway_sessions = self._live_gateway_sessions | frozenset((gateway_session_id,)) + + def release_upstream_sessions(self, gateway_session_id: str) -> None: + self._live_gateway_sessions = self._live_gateway_sessions - frozenset((gateway_session_id,)) + for key in tuple(key for key in self._upstream_sessions if key[0] == gateway_session_id): + self._upstream_sessions.pop(key).close() + for retiring_gateway_session_id, session in self._retiring_upstream_sessions: + if retiring_gateway_session_id == gateway_session_id: + session.close() + self._retiring_upstream_sessions = [ + (retiring_gateway_session_id, session) + for retiring_gateway_session_id, session in self._retiring_upstream_sessions + if retiring_gateway_session_id != gateway_session_id and not session.closed + ] + + async def close_all_upstream_sessions(self) -> None: + sessions: Final = ( + *self._upstream_sessions.values(), + *(session for _, session in self._retiring_upstream_sessions), + ) + self._upstream_sessions.clear() + self._retiring_upstream_sessions.clear() + self._live_gateway_sessions = frozenset() + for session in sessions: + session.close() + try: + await asyncio.wait_for( + asyncio.gather(*(session.wait_closed() for session in sessions)), + MCP_UPSTREAM_SESSION_SHUTDOWN_TIMEOUT_SECONDS, + ) + except asyncio.TimeoutError: + verbose_logger.warning("Timed out waiting for persistent MCP sessions to close during shutdown") + + def _drop_upstream_session(self, session: PersistentMCPSession | None) -> None: + matching_keys: Final = tuple(key for key, value in self._upstream_sessions.items() if value is session) + dropped_sessions: Final = tuple((key[0], self._upstream_sessions.pop(key)) for key in matching_keys) + for gateway_session_id, dropped in dropped_sessions: + dropped.close() + self._retiring_upstream_sessions.append((gateway_session_id, dropped)) + async def _obo_call_tool_with_retry( self, *, @@ -6138,27 +6257,33 @@ class MCPServerManager: raw_headers: Mapping[str, str] | None = None, client_ip: str | None = None, allow_input_required: bool = False, + persistent_session: PersistentMCPSession | None = None, ) -> CallToolResult | InputRequiredResult: """Call a token_exchange (OBO) tool; on an upstream 401/403 re-mint the token once and retry. The exchanged token is baked into the client at build time, so the retry invalidates the cached exchange and rebuilds the client (which re-exchanges). One retry only: a non-auth failure or a second auth failure degrades to the normal ``isError`` result, and a re-exchange - that now fails surfaces its own 401 challenge from ``_create_mcp_client``. + that now fails surfaces its own 401 challenge from ``create_mcp_client``. """ try: - return await client.call_tool( - call_tool_params, - host_progress_callback=host_progress_callback, - raise_on_error=True, - allow_input_required=allow_input_required, - ) + async with self._outbound_limit(mcp_server, persistent_session): + return await client.call_tool( + call_tool_params, + host_progress_callback=host_progress_callback, + raise_on_error=True, + allow_input_required=allow_input_required, + persistent_session=persistent_session, + ) except Exception as exc: - if _extract_upstream_auth_failure(exc) is None: + auth_failure: Final = _extract_upstream_auth_failure(exc) + if auth_failure is None and not isinstance(exc, UpstreamSessionClosedError): return MCPClient.error_tool_result(exc) - spec: Final = to_server_spec(mcp_server) - if spec is not None: - await self._cred_provider.invalidate_credentials(to_subject(user_api_key_auth, subject_token), spec) + if auth_failure is not None: + spec: Final = to_server_spec(mcp_server) + if spec is not None: + await self._cred_provider.invalidate_credentials(to_subject(user_api_key_auth, subject_token), spec) + self._drop_upstream_session(persistent_session) retry_client: Final = await self.create_mcp_client( server=mcp_server, mcp_auth_header=server_auth_header, @@ -6169,11 +6294,14 @@ class MCPServerManager: raw_headers=raw_headers, client_ip=client_ip, ) - return await retry_client.call_tool( - call_tool_params, - host_progress_callback=host_progress_callback, - allow_input_required=allow_input_required, - ) + retry_persistent_session: Final = await self._upstream_session_for(retry_client, mcp_server, raw_headers) + async with self._outbound_limit(mcp_server, retry_persistent_session): + return await retry_client.call_tool( + call_tool_params, + host_progress_callback=host_progress_callback, + allow_input_required=allow_input_required, + persistent_session=retry_persistent_session, + ) async def _call_regular_mcp_tool( self, @@ -6330,6 +6458,7 @@ class MCPServerManager: raw_headers=raw_headers, client_ip=client_ip, ) + persistent_session: Final = await self._upstream_session_for(client, mcp_server, raw_headers) call_tool_params: Final = MCPCallToolRequestParams( name=original_tool_name, @@ -6342,24 +6471,21 @@ class MCPServerManager: # OBO / ID-JAG: the exchanged token may have been revoked/rotated upstream since it was # cached, so an upstream 401 gets one invalidate + re-mint + retry. Gated to these modes; # all others keep the plain single call below. - async def _obo_call_tool_limited(): - async with self._limit_outbound_concurrency(mcp_server): - return await self._obo_call_tool_with_retry( - client=client, - call_tool_params=call_tool_params, - host_progress_callback=host_progress_callback, - mcp_server=mcp_server, - server_auth_header=server_auth_header, - extra_headers=extra_headers, - stdio_env=stdio_env, - subject_token=subject_token, - user_api_key_auth=user_api_key_auth, - raw_headers=raw_headers, - client_ip=client_ip, - allow_input_required=allow_input_required, - ) - - tool_call_coro = _obo_call_tool_limited() + tool_call_coro = self._obo_call_tool_with_retry( + client=client, + call_tool_params=call_tool_params, + host_progress_callback=host_progress_callback, + mcp_server=mcp_server, + server_auth_header=server_auth_header, + extra_headers=extra_headers, + stdio_env=stdio_env, + subject_token=subject_token, + user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, + allow_input_required=allow_input_required, + persistent_session=persistent_session, + ) else: # Scoped to the two client-forwarded token modes this stack introduced; legacy # oauth2 + delegate_auth_to_upstream (is_oauth_passthrough) is being removed, so it is not @@ -6368,12 +6494,13 @@ class MCPServerManager: server_label: Final = mcp_server.name or mcp_server.server_name or mcp_server.alias or "" async def _call_tool_via_client(client, params): - async with self._limit_outbound_concurrency(mcp_server): + async with self._outbound_limit(mcp_server, persistent_session): if not relays_upstream_auth: return await client.call_tool( params, host_progress_callback=host_progress_callback, allow_input_required=allow_input_required, + persistent_session=persistent_session, ) # The client-forwarded modes carry the caller's own upstream token, so an upstream # 401 (expired/invalid token) is the caller's to resolve: relay it as @@ -6390,6 +6517,7 @@ class MCPServerManager: host_progress_callback=host_progress_callback, raise_on_error=True, allow_input_required=allow_input_required, + persistent_session=persistent_session, ) except Exception as e: auth_info: Final = _extract_upstream_auth_failure(e) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index a30bf30e45f..d299f470f12 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -658,6 +658,7 @@ if MCP_AVAILABLE: _stateful_session_locks.pop(session_id, None) _stateful_session_active_request_counts.pop(session_id, None) _stateful_session_client_info.pop(session_id, None) + operations.global_mcp_server_manager.release_upstream_sessions(session_id) # Keep this alias so existing references to session_manager still work session_manager: Final = session_manager_stateless @@ -793,6 +794,7 @@ if MCP_AVAILABLE: _stateful_auth_context_cleanup_task.cancel() with contextlib.suppress(asyncio.CancelledError): await _stateful_auth_context_cleanup_task + await operations.global_mcp_server_manager.close_all_upstream_sessions() if _session_manager_stateful_cm: await _session_manager_stateful_cm.__aexit__(None, None, None) if _session_manager_cm: @@ -1264,13 +1266,16 @@ if MCP_AVAILABLE: return f"ip:{hashlib.sha256(client_ip.encode('utf-8')).hexdigest()}" return "anonymous" - def _peeked_json_object_body(body: bytes) -> bytes | None: - """The peeked bytes when they form a complete JSON-RPC object, else None so a - truncated prefix or a batch array fails closed and stays budget-enforced.""" + def _admission_auth_body(body: bytes) -> bytes: + """Return only the JSON-RPC method for admission auth.""" try: - return body if isinstance(json.loads(body), dict) else None - except (json.JSONDecodeError, TypeError): - return None + data: Final = TypeAdapter(dict[str, object]).validate_json(body) + except (ValueError, TypeError): + return b"{}" + method: Final = data.get("method") + if not isinstance(method, str): + return b"{}" + return json.dumps({"method": method}).encode() def _is_initialize_request(body: bytes) -> bool: """ @@ -2193,10 +2198,10 @@ if MCP_AVAILABLE: if peek is not None: receive = peek.receive # rebind-ok: replay peeked ASGI messages to the downstream handler - async def peeked_json_object_body() -> bytes: - return _peeked_json_object_body(await peek.admission_body()) or b"{}" + async def admission_auth_body() -> bytes: + return _admission_auth_body(await peek.admission_body()) - scope[MCP_PEEKED_BODY_SCOPE_KEY] = peeked_json_object_body + scope[MCP_PEEKED_BODY_SCOPE_KEY] = admission_auth_body ( user_api_key_auth, mcp_auth_header, @@ -2856,6 +2861,7 @@ if MCP_AVAILABLE: _stateful_session_auth_contexts[session_id] = auth_user _stateful_session_auth_context_last_seen[session_id] = time.monotonic() _stateful_session_owners[session_id] = owner_fingerprint + operations.global_mcp_server_manager.track_gateway_session(session_id) if client_info is not None: _stateful_session_client_info[session_id] = client_info break diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 69db4525a17..534796032c6 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -9494,17 +9494,878 @@ }, "decisions": { "components": { - "schemas": {} + "schemas": { + "ChoiceAnswer": { + "additionalProperties": true, + "properties": { + "choice": { + "title": "Choice", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "choice", + "confidence", + "probabilities" + ], + "title": "ChoiceAnswer", + "type": "object" + }, + "DecisionsResponse": { + "additionalProperties": true, + "properties": { + "answers": { + "additionalProperties": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/ChoiceAnswer", + "noul": "#/components/schemas/NoulAnswer", + "score": "#/components/schemas/ScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/NoulAnswer" + }, + { + "$ref": "#/components/schemas/ChoiceAnswer" + }, + { + "$ref": "#/components/schemas/ScoreAnswer" + } + ] + }, + "title": "Answers", + "type": "object" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/components/schemas/DecisionsUsage" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "answers" + ], + "title": "DecisionsResponse", + "type": "object" + }, + "DecisionsUsage": { + "additionalProperties": true, + "properties": { + "input_tokens": { + "default": 0, + "title": "Input Tokens", + "type": "integer" + }, + "output_tokens": { + "default": 0, + "title": "Output Tokens", + "type": "integer" + } + }, + "title": "DecisionsUsage", + "type": "object" + }, + "NoulAnswer": { + "additionalProperties": true, + "properties": { + "noul": { + "title": "Noul", + "type": "number" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "noul" + ], + "title": "NoulAnswer", + "type": "object" + }, + "OpenAIChoiceAnswer": { + "properties": { + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Choice" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probabilities": { + "items": { + "$ref": "#/components/schemas/OpenAIChoiceProbability" + }, + "title": "Probabilities", + "type": "array" + }, + "type": { + "const": "choice", + "default": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "choice", + "probabilities", + "confidence" + ], + "title": "OpenAIChoiceAnswer", + "type": "object" + }, + "OpenAIChoiceProbability": { + "properties": { + "probability": { + "title": "Probability", + "type": "number" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value", + "probability" + ], + "title": "OpenAIChoiceProbability", + "type": "object" + }, + "OpenAIDecisionInputTokensDetails": { + "properties": { + "cache_write_tokens": { + "default": 0, + "title": "Cache Write Tokens", + "type": "integer" + }, + "cached_tokens": { + "default": 0, + "title": "Cached Tokens", + "type": "integer" + } + }, + "title": "OpenAIDecisionInputTokensDetails", + "type": "object" + }, + "OpenAIDecisionOutputTokensDetails": { + "properties": { + "reasoning_tokens": { + "default": 0, + "title": "Reasoning Tokens", + "type": "integer" + } + }, + "title": "OpenAIDecisionOutputTokensDetails", + "type": "object" + }, + "OpenAIDecisionResponse": { + "additionalProperties": true, + "properties": { + "answers": { + "items": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/OpenAIChoiceAnswer", + "predicate": "#/components/schemas/OpenAIPredicateAnswer", + "refusal": "#/components/schemas/OpenAIRefusalAnswer", + "score": "#/components/schemas/OpenAIScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/OpenAIPredicateAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIChoiceAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIScoreAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIRefusalAnswer" + } + ] + }, + "title": "Answers", + "type": "array" + }, + "model": { + "title": "Model", + "type": "string" + }, + "usage": { + "$ref": "#/components/schemas/OpenAIDecisionUsage" + } + }, + "required": [ + "model", + "answers", + "usage" + ], + "title": "OpenAIDecisionResponse", + "type": "object" + }, + "OpenAIDecisionUsage": { + "properties": { + "input_tokens": { + "title": "Input Tokens", + "type": "integer" + }, + "input_tokens_details": { + "$ref": "#/components/schemas/OpenAIDecisionInputTokensDetails", + "default": { + "cache_write_tokens": 0, + "cached_tokens": 0 + } + }, + "output_tokens": { + "title": "Output Tokens", + "type": "integer" + }, + "output_tokens_details": { + "$ref": "#/components/schemas/OpenAIDecisionOutputTokensDetails", + "default": { + "reasoning_tokens": 0 + } + }, + "total_tokens": { + "title": "Total Tokens", + "type": "integer" + } + }, + "required": [ + "input_tokens", + "output_tokens", + "total_tokens" + ], + "title": "OpenAIDecisionUsage", + "type": "object" + }, + "OpenAIPredicateAnswer": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probability": { + "title": "Probability", + "type": "number" + }, + "type": { + "const": "predicate", + "default": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "probability" + ], + "title": "OpenAIPredicateAnswer", + "type": "object" + }, + "OpenAIRefusalAnswer": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "type": { + "const": "refusal", + "default": "refusal", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name" + ], + "title": "OpenAIRefusalAnswer", + "type": "object" + }, + "OpenAIScoreAnswer": { + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probabilities": { + "items": { + "$ref": "#/components/schemas/OpenAIScoreProbability" + }, + "title": "Probabilities", + "type": "array" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "default": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "score", + "probabilities", + "confidence" + ], + "title": "OpenAIScoreAnswer", + "type": "object" + }, + "OpenAIScoreProbability": { + "properties": { + "label": { + "title": "Label", + "type": "string" + }, + "probability": { + "title": "Probability", + "type": "number" + }, + "value": { + "title": "Value", + "type": "integer" + } + }, + "required": [ + "value", + "label", + "probability" + ], + "title": "OpenAIScoreProbability", + "type": "object" + }, + "ScoreAnswer": { + "additionalProperties": true, + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "legend": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "title": "Legend", + "type": "object" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "score", + "confidence", + "legend", + "probabilities" + ], + "title": "ScoreAnswer", + "type": "object" + } + } }, "paths": { "/decisions": { "post": { + "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these\nrules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_decisions_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "model": "luna", + "questions": [ + { + "instructions": "Is the customer asking for a refund?", + "name": "is_refund_request", + "type": "predicate" + }, + { + "choices": [ + { + "description": "can wait a week", + "value": "low" + }, + { + "description": "needs action today", + "value": "high" + } + ], + "instructions": "How urgent is this?", + "name": "urgency", + "type": "choice" + }, + { + "instructions": "How frustrated is the customer?", + "levels": [ + { + "label": "calm" + }, + { + "label": "mildly annoyed" + }, + { + "label": "angry" + } + ], + "name": "frustration", + "type": "score" + } + ] + }, + "schema": { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "input_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "title": "OpenAIDecisionInputText", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Detail" + }, + "image_url": { + "title": "Image Url", + "type": "string" + }, + "type": { + "const": "input_image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "image_url" + ], + "title": "OpenAIDecisionInputImage", + "type": "object" + } + ] + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "user", + "default": "user", + "title": "Role", + "type": "string" + }, + "type": { + "const": "message", + "default": "message", + "title": "Type", + "type": "string" + } + }, + "required": [ + "content" + ], + "title": "OpenAIDecisionInputMessage", + "type": "object" + }, + "type": "array" + } + ], + "description": "The text being judged, or user messages whose content mixes input_text and input_image parts", + "title": "Input" + }, + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "description": "Predicate, choice and score questions. Answers come back in the same order", + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The yes/no question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "OpenAIPredicateQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "choices": { + "description": "The options the model picks from. Values must be unique", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "OpenAIChoiceOption", + "type": "object" + }, + "maxItems": 255, + "minItems": 2, + "title": "Choices", + "type": "array" + }, + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "choices" + ], + "title": "OpenAIChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "levels": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "label" + ], + "title": "OpenAIScoreLevel", + "type": "object" + }, + "maxItems": 10, + "minItems": 2, + "title": "Levels", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "levels" + ], + "title": "OpenAIScoreQuestion", + "type": "object" + } + ] + }, + "maxItems": 128, + "minItems": 1, + "title": "Questions", + "type": "array" + }, + "safety_identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A stable id for your end user. Sent to OpenAI and dropped for other providers", + "title": "Safety Identifier" + } + }, + "required": [ + "input", + "questions", + "model" + ], + "title": "OpenAIDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/OpenAIDecisionResponse" + } } }, "description": "Successful Response" @@ -9515,7 +10376,7 @@ "APIKeyHeader": [] } ], - "summary": "Decisions", + "summary": "Ask questions about an input (OpenAI Decisions format)", "tags": [ "decisions" ] @@ -9523,12 +10384,290 @@ }, "/systemone": { "post": { + "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider\nis called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "jev", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "urgency": { + "criteria": { + "high": "needs action today", + "low": "can wait a week" + }, + "instructions": "How urgent is this?", + "type": "choice" + } + }, + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + }, + "schema": { + "additionalProperties": true, + "properties": { + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": true, + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what makes the answer true and what makes it false", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The yes/no question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "NoulQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "description": "Candidate labels mapped to an optional description of each", + "maxProperties": 255, + "minProperties": 1, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "maxItems": 10, + "minItems": 1, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ScoreQuestion", + "type": "object" + } + ] + }, + "description": "Named noul, choice and score questions. Each key becomes a key in the answers", + "maxProperties": 128, + "minProperties": 1, + "propertyNames": { + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The thing being judged, such as a support ticket, a document or a chat transcript", + "title": "State" + } + }, + "required": [ + "state", + "questions", + "model" + ], + "title": "DecisionsRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/DecisionsResponse" + } } }, "description": "Successful Response" @@ -9539,7 +10678,7 @@ "APIKeyHeader": [] } ], - "summary": "Systemone", + "summary": "Ask named questions about a state (System One format)", "tags": [ "decisions" ] @@ -9547,12 +10686,382 @@ }, "/v1/decisions": { "post": { + "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these\nrules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_v1_decisions_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "model": "luna", + "questions": [ + { + "instructions": "Is the customer asking for a refund?", + "name": "is_refund_request", + "type": "predicate" + }, + { + "choices": [ + { + "description": "can wait a week", + "value": "low" + }, + { + "description": "needs action today", + "value": "high" + } + ], + "instructions": "How urgent is this?", + "name": "urgency", + "type": "choice" + }, + { + "instructions": "How frustrated is the customer?", + "levels": [ + { + "label": "calm" + }, + { + "label": "mildly annoyed" + }, + { + "label": "angry" + } + ], + "name": "frustration", + "type": "score" + } + ] + }, + "schema": { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "input_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "title": "OpenAIDecisionInputText", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Detail" + }, + "image_url": { + "title": "Image Url", + "type": "string" + }, + "type": { + "const": "input_image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "image_url" + ], + "title": "OpenAIDecisionInputImage", + "type": "object" + } + ] + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "user", + "default": "user", + "title": "Role", + "type": "string" + }, + "type": { + "const": "message", + "default": "message", + "title": "Type", + "type": "string" + } + }, + "required": [ + "content" + ], + "title": "OpenAIDecisionInputMessage", + "type": "object" + }, + "type": "array" + } + ], + "description": "The text being judged, or user messages whose content mixes input_text and input_image parts", + "title": "Input" + }, + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "description": "Predicate, choice and score questions. Answers come back in the same order", + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The yes/no question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "OpenAIPredicateQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "choices": { + "description": "The options the model picks from. Values must be unique", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "OpenAIChoiceOption", + "type": "object" + }, + "maxItems": 255, + "minItems": 2, + "title": "Choices", + "type": "array" + }, + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "choices" + ], + "title": "OpenAIChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "levels": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "label" + ], + "title": "OpenAIScoreLevel", + "type": "object" + }, + "maxItems": 10, + "minItems": 2, + "title": "Levels", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "levels" + ], + "title": "OpenAIScoreQuestion", + "type": "object" + } + ] + }, + "maxItems": 128, + "minItems": 1, + "title": "Questions", + "type": "array" + }, + "safety_identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A stable id for your end user. Sent to OpenAI and dropped for other providers", + "title": "Safety Identifier" + } + }, + "required": [ + "input", + "questions", + "model" + ], + "title": "OpenAIDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/OpenAIDecisionResponse" + } } }, "description": "Successful Response" @@ -9563,7 +11072,7 @@ "APIKeyHeader": [] } ], - "summary": "Decisions", + "summary": "Ask questions about an input (OpenAI Decisions format)", "tags": [ "decisions" ] @@ -9571,12 +11080,290 @@ }, "/v1/systemone": { "post": { + "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider\nis called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_v1_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "jev", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "urgency": { + "criteria": { + "high": "needs action today", + "low": "can wait a week" + }, + "instructions": "How urgent is this?", + "type": "choice" + } + }, + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + }, + "schema": { + "additionalProperties": true, + "properties": { + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": true, + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what makes the answer true and what makes it false", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The yes/no question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "NoulQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "description": "Candidate labels mapped to an optional description of each", + "maxProperties": 255, + "minProperties": 1, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "maxItems": 10, + "minItems": 1, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ScoreQuestion", + "type": "object" + } + ] + }, + "description": "Named noul, choice and score questions. Each key becomes a key in the answers", + "maxProperties": 128, + "minProperties": 1, + "propertyNames": { + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The thing being judged, such as a support ticket, a document or a chat transcript", + "title": "State" + } + }, + "required": [ + "state", + "questions", + "model" + ], + "title": "DecisionsRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/DecisionsResponse" + } } }, "description": "Successful Response" @@ -9587,7 +11374,7 @@ "APIKeyHeader": [] } ], - "summary": "Systemone", + "summary": "Ask named questions about a state (System One format)", "tags": [ "decisions" ] diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0715a4cae97..b6c81f17761 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3,7 +3,7 @@ import json import os from collections.abc import Callable, Mapping, Sequence from datetime import datetime -from types import MappingProxyType +from types import MappingProxyType as MappingProxyType # noqa: PLC0414 # public re-export from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NamedTuple, TypeAlias import httpx @@ -16,7 +16,6 @@ from pydantic import ( Json, JsonValue, PositiveInt, - PrivateAttr, TypeAdapter, field_validator, model_validator, @@ -26,19 +25,24 @@ from typing_extensions import ( ReadOnly, Required, TypedDict, - TypeIs, # noqa: TID251 # narrows the before-validator input without copying it ) from litellm._uuid import uuid from litellm.constants import DEFAULT_STAGGER_WINDOW_SECONDS, MCP_STDIO_ALLOWED_COMMANDS from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( validate_arize_otlp_protocol_value, + validate_capture_message_content_value, validate_langfuse_environment_value, validate_langfuse_span_scope_value, validate_no_callback_env_reference, ) from litellm.proxy._experimental.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled -from litellm.types.agents import AgentCaller, AgentResponse +from litellm.types.agents import ( + AgentCaller as AgentCaller, # noqa: PLC0414 # public re-export +) +from litellm.types.agents import ( + AgentResponse as AgentResponse, # noqa: PLC0414 # public re-export +) from litellm.types.integrations.compression_interception import ( CompressionSavingsMetadata, ) @@ -67,11 +71,17 @@ from litellm.types.proxy.agent_identity import ManagedAgentContext from litellm.types.proxy.auth.special_headers import ( SpecialHeaders as SpecialHeaders, # noqa: PLC0414 # public re-export ) -from litellm.types.proxy.auth.user_api_key_auth import is_jwt +from litellm.types.proxy.auth.user_api_key_auth import ( + is_jwt as is_jwt, # noqa: PLC0414 # public re-export +) from litellm.types.proxy.carried_budget_state import ( - OrgBudgetSnapshot, - TeamBudgetSnapshot, - UserBudgetSnapshot, + OrgBudgetSnapshot as OrgBudgetSnapshot, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.carried_budget_state import ( + TeamBudgetSnapshot as TeamBudgetSnapshot, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.carried_budget_state import ( + UserBudgetSnapshot as UserBudgetSnapshot, # noqa: PLC0414 # public re-export ) from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry from litellm.types.proxy.spend_capture_rate import SpendCaptureRateCheckSettings @@ -79,6 +89,7 @@ from litellm.types.router import AllowedModelRegion, RouterErrors, UpdateRouterC from litellm.types.router_weights import validate_router_settings_dict from litellm.types.secret_managers.main import KeyManagementSystem from litellm.types.utils import ( + CAPTURE_MESSAGE_CONTENT_VAR, AzureSpillover, CallTypes, CostBreakdown, @@ -196,17 +207,9 @@ class LitellmTableNames(str, enum.Enum): from litellm.types.integrations.slack_alerting import ( Litellm_EntityType as Litellm_EntityType, # noqa: E402, PLC0414 # public re-export ) - - -def hash_token(token: str) -> str: - import hashlib - - # Hash the string using SHA-256 - hashed_token: Final = hashlib.sha256(token.encode()).hexdigest() - - return hashed_token - - +from litellm.types.proxy.auth.user_api_key_auth import ( # noqa: E402 # re-export after the definitions above + hash_token as hash_token, # noqa: PLC0414 # public re-export +) from litellm.types.proxy.auth.user_roles import ( # noqa: E402 # re-export after the definitions above KeyManagementRoutes as KeyManagementRoutes, # noqa: PLC0414 # public re-export ) @@ -1163,6 +1166,18 @@ from litellm.models.team import BudgetLimitEntry as BudgetLimitEntry # noqa: E4 from litellm.types.object_permission import ( # noqa: E402 ObjectPermissionDict as ObjectPermissionDict, ) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + AllowedVectorStoreIndexItem as AllowedVectorStoreIndexItem, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + GenerateKeyRequest as GenerateKeyRequest, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + KeyRequestBase as KeyRequestBase, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + LiteLLMKeyType as LiteLLMKeyType, # noqa: PLC0414 # public re-export +) from litellm.types.proxy.management_endpoints.request_base import ( # noqa: E402 # re-export after the definitions above GenerateRequestBase as GenerateRequestBase, # noqa: PLC0414 # public re-export ) @@ -1171,71 +1186,6 @@ from litellm.types.proxy.management_endpoints.request_base import ( # noqa: E40 ) -class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase): - index_name: str - index_permissions: list[Literal["read", "write"]] - - -class KeyRequestBase(GenerateRequestBase): - key: str | None = None - tpd_limit: int | None = None - default_estimated_output_tokens: PositiveInt | None = None - default_estimated_output_tokens_per_model: Mapping[str, PositiveInt] | None = None - budget_id: str | None = None - end_user_budget_id: str | None = None - tags: list[str] | None = None - disable_global_guardrails: bool | None = None - enable_prompt_caching: bool | None = None - throttle_on_budget_exceeded: bool | None = None - enforced_params: list[str] | None = None - allowed_routes: list[Any] | None = [] - allowed_passthrough_routes: list[object] | None = None - denied_passthrough_routes: list[str] | None = None - allowed_vector_store_indexes: list[AllowedVectorStoreIndexItem] | None = None - rpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( - None # raise an error if 'guaranteed_throughput' is set and we're overallocating rpm - ) - tpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( - None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm - ) - router_settings: UpdateRouterConfig | None = None - access_group_ids: list[str] | None = None - - -class LiteLLMKeyType(str, enum.Enum): - """ - Enum for key types that determine what routes a key can access - """ - - LLM_API = "llm_api" # Can call LLM API routes (chat/completions, embeddings, etc.) - MANAGEMENT = "management" # Can call management routes (user/team/key management) - READ_ONLY = "read_only" # Can only call info/read routes - DEFAULT = "default" # Uses default allowed routes - - -class GenerateKeyRequest(KeyRequestBase): - soft_budget: float | None = None - send_invite_email: bool | None = None - key_type: LiteLLMKeyType | None = Field( - default=LiteLLMKeyType.DEFAULT, - description="Type of key that determines default allowed routes.", - ) - auto_rotate: bool | None = Field(default=False, description="Whether this key should be automatically rotated") - rotation_interval: str | None = Field( - default=None, - description="How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True", - ) - organization_id: str | None = None - project_id: str | None = None - - @field_validator("team_id", "organization_id", "project_id", mode="before") - @classmethod - def treat_cleared_id_as_unset(cls, v: object) -> object: - if v == "": - return None - return v - - class GenerateKeyResponse(KeyRequestBase): key: str key_name: str | None = None @@ -1278,65 +1228,12 @@ class GenerateKeyResponse(KeyRequestBase): return values -def _is_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: raw before-validator payload - return isinstance(value, Mapping) - - -class UpdateKeyRequest(KeyRequestBase): - # Note: the defaults of all Params here MUST BE NONE - # else they will get overwritten - duration: str | None = None - spend: float | None = None - soft_budget: float | None = None - metadata: dict[Any, Any] | None = None - temp_budget_increase: float | None = None - temp_budget_expiry: datetime | None = None - auto_rotate: bool | None = None - rotation_interval: str | None = None - organization_id: str | None = None - - project_id: str | None = Field( - default=None, - description="Omit to retain the project, or send null to detach. Assigning a different project is not supported.", - ) - - @model_validator(mode="before") - @classmethod - def drop_blank_team_id(cls, values: object) -> object: - if _is_mapping(values) and values.get("team_id") == "": - return MappingProxyType({k: v for k, v in values.items() if k != "team_id"}) - return values - - @field_validator("organization_id", mode="before") - @classmethod - def treat_cleared_organization_id_as_unset(cls, v: object) -> object: - if v == "": - return None - return v - - @model_validator(mode="after") - def validate_temp_budget(self) -> "UpdateKeyRequest": - if self.temp_budget_increase is not None or self.temp_budget_expiry is not None: - if self.temp_budget_increase is None or self.temp_budget_expiry is None: - raise ValueError("temp_budget_increase and temp_budget_expiry must be set together") - return self - - @model_validator(mode="after") - def validate_key_identifier(self) -> "UpdateKeyRequest": - if self.key is None and self.key_alias is None: - raise ValueError("either key or key_alias must be provided") - return self - - -class RegenerateKeyRequest(GenerateKeyRequest): - # This needs to be different from UpdateKeyRequest, because "key" is optional for this - key: str | None = None - new_key: str | None = None - duration: str | None = None - spend: float | None = None - metadata: dict[Any, Any] | None = None - new_master_key: str | None = None - grace_period: str | None = None # Duration to keep old key valid (e.g. "24h", "2d"); None = immediate revoke +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + RegenerateKeyRequest as RegenerateKeyRequest, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + UpdateKeyRequest as UpdateKeyRequest, # noqa: PLC0414 # public re-export +) class ResetSpendRequest(LiteLLMPydanticObjectBase): @@ -1788,41 +1685,12 @@ class ListSkillsRequest(LiteLLMPydanticObjectBase): offset: int | None = 0 -class NewUserRequestTeam(LiteLLMPydanticObjectBase): - team_id: str - max_budget_in_team: float | None = None - user_role: Literal["user", "admin"] = "user" - - -class NewUserRequest(GenerateRequestBase): - max_budget: float | None = None - user_email: str | None = None - user_alias: str | None = None - user_role: ( - Literal[ - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - LitellmUserRoles.INTERNAL_USER, - LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, - ] - | None - ) = None - teams: list[str] | list[NewUserRequestTeam] | None = None - auto_create_key: bool = True # flag used for returning a key as part of the /user/new response - send_invite_email: bool | None = None - sso_user_id: str | None = None - organizations: list[str] | None = None - password: str | None = None - - @field_validator("password") - @classmethod - def password_not_supported(cls, value: str | None) -> str | None: - if value is not None: - raise ValueError( - "password cannot be set via /user/new. Users set their own password through an " - "invitation link (POST /invitation/new)." - ) - return value +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + NewUserRequest as NewUserRequest, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + NewUserRequestTeam as NewUserRequestTeam, # noqa: PLC0414 # public re-export +) class NewUserResponse(GenerateKeyResponse): @@ -1844,36 +1712,12 @@ class NewUserResponse(GenerateKeyResponse): updated_at: datetime | None = None -class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase): # shared with BulkUpdateUserRequest - # repr=False keeps the plaintext out of management-endpoint alerts, which str() the request model - password: str | None = Field(default=None, repr=False) - spend: float | None = None - metadata: dict[object, object] | None = None - user_alias: str | None = None - user_role: ( - Literal[ - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - LitellmUserRoles.INTERNAL_USER, - LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, - ] - | None - ) = None - max_budget: float | None = None - - -class UpdateUserRequest(UpdateUserRequestNoUserIDorEmail): - # Note: the defaults of all Params here MUST BE NONE - # else they will get overwritten - user_id: str | None = None - user_email: str | None = None - - @model_validator(mode="before") - @classmethod - def check_user_info(cls, values: Mapping[str, object]) -> Mapping[str, object]: - if values.get("user_id") is None and values.get("user_email") is None: - raise ValueError("Either user id or user email must be provided") - return values +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + UpdateUserRequest as UpdateUserRequest, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + UpdateUserRequestNoUserIDorEmail as UpdateUserRequestNoUserIDorEmail, # noqa: PLC0414 # public re-export +) class ChangePasswordRequest(LiteLLMPydanticObjectBase): @@ -2202,7 +2046,7 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): @classmethod def validate_callback_vars(cls, values): callback_vars: Final = values.get("callback_vars", {}) - valid_keys: Final = set(StandardCallbackDynamicParams.__annotations__.keys()) + valid_keys: Final = {*StandardCallbackDynamicParams.__annotations__, CAPTURE_MESSAGE_CONTENT_VAR} for key, value in callback_vars.items(): if key not in valid_keys: raise ValueError(f"Invalid callback variable: {key}. Must be one of {valid_keys}") @@ -2214,6 +2058,8 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): validate_langfuse_span_scope_value(callback_vars[key]) if key == "arize_otlp_protocol": validate_arize_otlp_protocol_value(callback_vars[key]) + if key == CAPTURE_MESSAGE_CONTENT_VAR: + validate_capture_message_content_value(callback_vars[key]) return values @@ -3213,296 +3059,15 @@ from litellm.models.verification_token import ( # noqa: E402 from litellm.models.verification_token import ( # noqa: E402 LiteLLM_VerificationToken as LiteLLM_VerificationToken, ) - -_UntypedDict: TypeAlias = dict[Any, Any] - - -def _is_str_keyed_dict(value: object) -> TypeIs[dict[str, object]]: # guard-ok: pydantic input and JSON keys are str - return isinstance(value, dict) - - -class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): - """ - Combined view of litellm verification token + litellm team table (select values) - """ - - team_spend: float | None = None - team_alias: str | None = None - team_tpm_limit: int | None = None - team_rpm_limit: int | None = None - team_tpd_limit: int | None = None - team_max_budget: float | None = None - team_soft_budget: float | None = None - team_model_max_budget: dict[str, object] | None = None - team_models: list[Any] = [] - team_blocked: bool = False - soft_budget: float | None = None - team_model_aliases: _UntypedDict | None = None - team_member: Member | None = None - team_metadata: _UntypedDict | None = None - team_object_permission_id: str | None = None - - # Team Member Specific Params - team_member_spend: float | None = None - team_member_tpm_limit: int | None = None - team_member_rpm_limit: int | None = None - - # End User Params - end_user_id: str | None = None - end_user_tpm_limit: int | None = None - end_user_rpm_limit: int | None = None - end_user_tpd_limit: int | None = None - end_user_max_budget: float | None = None - end_user_model_max_budget: _UntypedDict | None = None - - # Organization Params - organization_alias: str | None = None - organization_max_budget: float | None = None - organization_tpm_limit: int | None = None - organization_rpm_limit: int | None = None - organization_metadata: _UntypedDict | None = None - - # Project Params - project_alias: str | None = None - project_metadata: _UntypedDict | None = None - - # Time stamps - last_refreshed_at: float | None = None # last time joint view was pulled from db - - def __init__(self, **kwargs: object) -> None: - # Handle litellm_budget_table_* keys (budget table overrides when key value is None or empty) - for key, value in list(kwargs.items()): - if key.startswith("litellm_budget_table_") and value is not None: - # Extract the corresponding attribute name - attr_name = key.replace("litellm_budget_table_", "") - # Use key's value from kwargs (from DB view), not class default - current = kwargs.get(attr_name) - if current is None: - current = getattr(self, attr_name, None) - # Apply budget value when key has no value, or for model_max_budget when key has empty dict - should_apply = current is None or ( - attr_name == "model_max_budget" and _is_str_keyed_dict(current) and len(current) == 0 - ) - if should_apply: - kwargs[attr_name] = value - if key == "end_user_id" and value is not None and isinstance(value, int): - kwargs[key] = str(value) - - if kwargs.get("organization_id") is not None: - kwargs["org_id"] = kwargs.pop("organization_id") - # Initialize the superclass - super().__init__(**kwargs) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # pydantic validates raw values - - -class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response object for user api key auth - """ - Return the row in the db - """ - - api_key: str | None = None - user_role: LitellmUserRoles | None = None - allowed_model_region: AllowedModelRegion | None = None - parent_otel_span: Span | None = None - rpm_limit_per_model: dict[str, int] | None = None - tpm_limit_per_model: dict[str, int] | None = None - user_tpm_limit: int | None = None - user_rpm_limit: int | None = None - user_email: str | None = None - user_spend: float | None = None - user_max_budget: float | None = None - # Values stay `object` rather than BudgetConfig: this is the raw JSON column, - # and validating it here would make one malformed row fail auth outright. - # resolve_model_budget validates the single entry a request actually needs. - user_model_max_budget: Mapping[str, object] | None = None - team_member_model_max_budget: Mapping[str, object] | None = Field(default=None, exclude=True) - request_route: str | None = None - is_session_token: bool = False - # Server-only marker set exclusively by the MCP gateway admission path - # (reload_admitted_user) for a keyless user-subject admitted via a gateway DCR session - # bearer or bridge envelope. Not a DB column and never populated from caller-controlled key - # metadata or JWT claims, so it cannot be forged to gain the team-inherited MCP grant union - # or to escape the caller-Authorization egress scrub. exclude=True keeps it out of serialization. - mcp_admitted_user_subject: bool = Field(default=False, exclude=True) - requires_fresh_policy: bool = Field(default=False, exclude=True) - mcp_explicit_grants_only: bool = Field(default=False, exclude=True) - # team_id -> that team's mcp_rpm_limit map, for a keyless admitted subject that reaches MCP - # servers through several teams at once and therefore has no single team_id for the limiter to - # key off. Server-only and stripped from validated input for the same reason as the marker - # above: a forged entry would let a caller pick which team's rpm bucket it is charged against. - mcp_source_team_rpm_limits: dict[str, dict[str, int]] | None = Field(default=None, exclude=True) - # The single MCP server_id a gateway session bearer was scoped to at authorize time (RFC 8707 - # resource), or None for an aggregate-scope session. A RESTRICTION intersected against the live - # grant resolution, never a grant. Server-only, set exclusively by the MCP gateway admission - # path via post-construction assignment and stripped from validated input like the markers - # above; a forged value could at most narrow, but the stripping keeps the field's provenance - # single-owner so its meaning stays trustworthy. - mcp_session_resource_server_id: str | None = Field(default=None, exclude=True) - mcp_toolset_id: str | None = Field(default=None, exclude=True) - authenticated_by_custom_auth: bool = Field(default=False, exclude=True) - via_virtual_key: bool = Field( - default=False, - exclude=True, - description=( - "Server-only marker set exclusively by the DB virtual-key and master-key auth paths via " - "post-construction assignment. Stripped from validated input so custom auth handlers, JWT " - "claims, or key metadata cannot forge it. Gates overwrite_user_with_key_hash stamping: only " - "a credential the proxy itself validated as a key may be forwarded as the provider-facing " - "user id." - ), - ) - invoked_agent_id: str | None = Field(default=None, exclude=True) - invoked_agent_policy: AgentResponse | None = Field(default=None, exclude=True) - agent_invocation_cost: float | None = Field(default=None, exclude=True) - billing_agent_policy: AgentResponse | None = Field(default=None, exclude=True) - _managed_delegation_verified: bool = PrivateAttr(default=False) - managed_agent_policy: AgentResponse | None = Field(default=None, exclude=True) - managed_agent_context: ManagedAgentContext | None = Field(default=None, exclude=True) - agent_caller: AgentCaller | None = Field( - default=None, - exclude=True, - description=( - "Set per request from the x-litellm-user-id / x-litellm-team-id headers an agent echoes back on " - "calls made with its own key. Every check treats it as a ceiling, so a forged value can only " - "narrow the agent's access." - ), - ) - budget_reservation: dict[str, Any] | None = Field(default=None, exclude=True) - team_budget_snapshot: TeamBudgetSnapshot | None = Field(default=None, exclude=True) - user_budget_snapshot: UserBudgetSnapshot | None = Field(default=None, exclude=True) - org_budget_snapshot: OrgBudgetSnapshot | None = Field(default=None, exclude=True) - matched_model_access_groups: list[str] | None = Field(default=None, exclude=True) - budget_throttle_pct: float | None = Field(default=None, exclude=True) - user: Any | None = None # Expanded user object when expand=user is used - created_by_user: Any | None = None # Expanded created_by user when expand=user is used - end_user_object_permission: LiteLLM_ObjectPermissionTable | None = None - # Team object_permission preloaded in auth (e.g. get_team_object) to avoid - # per-request object_permission fetches in downstream checks (vector stores, etc.) - team_object_permission: LiteLLM_ObjectPermissionTable | None = None - # Decoded upstream IdP claims (groups, roles, etc.) propagated by JWT auth machinery - # and forwarded into outbound tokens by guardrails such as MCPJWTSigner. - jwt_claims: _UntypedDict | None = None - - model_config = ConfigDict(arbitrary_types_allowed=True) - - @model_validator(mode="before") - @classmethod - def check_api_key(cls, values: object) -> object: - # If values is already an instance (not a dict), return it as-is - if not _is_str_keyed_dict(values): - return values - # mcp_admitted_user_subject is a server-only marker, set ONLY by the MCP gateway admission - # path via post-construction assignment. Strip it from any validated input (constructor - # kwargs, model_validate, a JWT/key claim splat) so it can never be forged from caller data. - values.pop("mcp_admitted_user_subject", None) - values.pop("requires_fresh_policy", None) - values.pop("mcp_explicit_grants_only", None) - values.pop("mcp_source_team_rpm_limits", None) - values.pop("mcp_session_resource_server_id", None) - values.pop("mcp_toolset_id", None) - values.pop("via_virtual_key", None) - values.pop("authenticated_by_custom_auth", None) - values.pop("agent_caller", None) - values.pop("managed_agent_context", None) - values.pop("managed_agent_policy", None) - values.pop("invoked_agent_id", None) - values.pop("invoked_agent_policy", None) - values.pop("agent_invocation_cost", None) - values.pop("billing_agent_policy", None) - api_key: Final = values.get("api_key") - if api_key is not None: - values.update({"token": cls._safe_hash_litellm_api_key(api_key)}) # pyright: ignore[reportArgumentType] # non-str fails in the hasher - if isinstance(api_key, str): - values.update({"api_key": cls._safe_hash_litellm_api_key(api_key)}) - return values - - @classmethod - def _safe_hash_litellm_api_key(cls, api_key: str) -> str: - """ - Helper to ensure all logged keys are hashed - Covers: - 1. Regular API keys from LiteLLM DB - 2. JWT tokens used for connecting to LiteLLM API - """ - normalized = api_key - if normalized[:7].lower() == "bearer ": - normalized = normalized[7:] - if normalized.startswith("sk-"): - return hash_token(normalized) - if is_jwt(token=normalized): - return f"hashed-jwt-{hash_token(token=normalized)}" - return normalized - - @classmethod - def get_litellm_internal_health_check_user_api_key_auth(cls) -> "UserAPIKeyAuth": - """ - Returns a `UserAPIKeyAuth` object for the litellm internal health check service account. - - This is used to track number of requests/spend for health check calls. - """ - from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME - - return cls( - api_key=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, - team_id=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, - key_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, - team_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, - ) - - @classmethod - def get_litellm_cli_user_api_key_auth(cls) -> "UserAPIKeyAuth": - """ - Returns a `UserAPIKeyAuth` object for the litellm internal health check service account. - - This is used to track number of requests/spend for health check calls. - """ - from litellm.constants import LITTELM_CLI_SERVICE_ACCOUNT_NAME - - return cls( - api_key=LITTELM_CLI_SERVICE_ACCOUNT_NAME, - team_id=LITTELM_CLI_SERVICE_ACCOUNT_NAME, - key_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, - team_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, - ) - - @classmethod - def get_litellm_internal_jobs_user_api_key_auth(cls) -> "UserAPIKeyAuth": - """ - Returns a `UserAPIKeyAuth` object for internal LiteLLM jobs like key rotation. - - This is used to track actions performed by automated system jobs. - """ - from litellm.constants import LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME - - return cls( - api_key=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, - team_id="system", - key_alias=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, - team_alias="system", - user_id="system", - user_role=LitellmUserRoles.PROXY_ADMIN, - ) - - @property - def is_team_service_account(self) -> bool: - return ( - self.user_id is None - and self.team_id is not None - and bool(self.metadata) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped base field - and self.metadata.get("service_account_id") is not None # pyright: ignore[reportUnknownMemberType] # untyped base field - ) - - -def user_api_key_has_admin_view(user_api_key_dict: UserAPIKeyAuth) -> bool: - """Return True if the caller's role grants unscoped read access to all - tenant resources (managed files, batches, vector stores, spend rows, etc). - - Lives on _types.py so leaf modules (e.g. litellm.llms.base_llm.managed_resources) - can use it without pulling in litellm.proxy.utils via management_endpoints. - """ - return user_api_key_dict.user_role in ( - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - ) +from litellm.types.proxy.auth.user_api_key_auth import ( # noqa: E402 # re-export after the definitions above + LiteLLM_VerificationTokenView as LiteLLM_VerificationTokenView, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.auth.user_api_key_auth import ( + UserAPIKeyAuth as UserAPIKeyAuth, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.auth.user_api_key_auth import ( + user_api_key_has_admin_view as user_api_key_has_admin_view, # noqa: PLC0414 # public re-export +) class UserInfoResponse(LiteLLMPydanticObjectBase): @@ -3734,10 +3299,6 @@ class LiteLLM_UserTableFiltered(LiteLLMBaseModel): # done to avoid exposing sen user_email: str | None = None -class LiteLLM_UserTableWithKeyCount(LiteLLM_UserTable): - key_count: int = 0 - - from litellm.models.access_group import ( # noqa: E402 LiteLLM_AccessGroupTable as LiteLLM_AccessGroupTable, ) @@ -3751,6 +3312,9 @@ from litellm.models.spend_logs import ( # noqa: E402 LiteLLM_SpendLogs as LiteLLM_SpendLogs, ) from litellm.models.tag import LiteLLM_TagTable as LiteLLM_TagTable # noqa: E402 +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + LiteLLM_UserTableWithKeyCount as LiteLLM_UserTableWithKeyCount, # noqa: PLC0414 # public re-export +) AUDIT_ACTIONS = Literal["created", "updated", "deleted", "blocked", "unblocked", "rotated", "kill_switch_fired"] @@ -4526,16 +4090,9 @@ class OrganizationAddMemberResponse(LiteLLMPydanticObjectBase): updated_organization_memberships: list[LiteLLM_OrganizationMembershipTable] -class MemberDeleteRequest(LiteLLMPydanticObjectBase): - user_id: str | None = None - user_email: str | None = None - - @model_validator(mode="before") - @classmethod - def check_user_info(cls, values: Mapping[str, object]) -> Mapping[str, object]: - if values.get("user_id") is None and values.get("user_email") is None: - raise ValueError("Either user id or user email must be provided") - return values +from litellm.types.proxy.management_endpoints.team_endpoints import ( # noqa: E402 # re-export after the definitions above + MemberDeleteRequest as MemberDeleteRequest, # noqa: PLC0414 # public re-export +) class MemberUpdateResponse(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/common_utils/callback_config_validation.py b/litellm/proxy/common_utils/callback_config_validation.py index 5b9fb2e3977..4956da853c6 100644 --- a/litellm/proxy/common_utils/callback_config_validation.py +++ b/litellm/proxy/common_utils/callback_config_validation.py @@ -10,6 +10,8 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Final +from litellm.types.utils import CAPTURE_MESSAGE_CONTENT_VAR + _NEWRELIC_CALLBACK: Final = "newrelic" _NEWRELIC_VAR_PREFIX: Final = "newrelic_" _LANGFUSE_OTEL_CALLBACK: Final = "langfuse_otel" @@ -36,6 +38,9 @@ def callback_config_error( ) if langfuse_error is not None: return langfuse_error + capture_error: Final = _capture_message_content_error(callback_name, callback_vars, callback_type) + if capture_error is not None: + return capture_error if callback_name != _NEWRELIC_CALLBACK: return None return _newrelic_config_error(callback_vars) @@ -80,6 +85,32 @@ def _langfuse_span_scope_error(callback_name: str | None, callback_vars: Mapping return None +def _capture_message_content_error( + callback_name: str | None, callback_vars: Mapping[str, str], callback_type: str | None +) -> str | None: + value: Final = callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR) + if value is None: + return None + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.integrations.otel.presets.destinations import destination_capable_backends + from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + validate_capture_message_content_value, + ) + + supported: Final = sorted(destination_capable_backends()) + if callback_name not in supported: + return f"{CAPTURE_MESSAGE_CONTENT_VAR} applies to the OTel v2 callbacks {supported} only, not {callback_name!r}" + if not is_otel_v2_enabled(): + return f"Per-destination {CAPTURE_MESSAGE_CONTENT_VAR} requires the proxy to run with LITELLM_OTEL_V2=true." + try: + validate_capture_message_content_value(value) + except ValueError as e: + return str(e) + if callback_type == "failure": + return f"{CAPTURE_MESSAGE_CONTENT_VAR} needs callback_type 'success' or 'success_and_failure'" + return None + + # Which credential family a dynamic variable belongs to. The families are the # integrations that share one account: every langfuse_* variable configures the # same Langfuse project whether it rides the classic callback or the OTel one, @@ -190,20 +221,41 @@ def conflicting_span_scope_error( ) +def conflicting_capture_error( + callback_name: str, + callback_vars: Mapping[str, str] | None, + stored_entries: Sequence[tuple[str | None, Mapping[str, str]]], +) -> str | None: + incoming: Final = None if callback_vars is None else callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR) + if incoming is None: + return None + return next( + ( + f"{CAPTURE_MESSAGE_CONTENT_VAR} is already set to {stored!r} by another {callback_name} entry. " + f"Every {callback_name} entry shares one value: remove that entry or send the same value." + for name, entry_vars in stored_entries + if name == callback_name and (stored := entry_vars.get(CAPTURE_MESSAGE_CONTENT_VAR)) not in (None, incoming) + ), + None, + ) + + def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str | None: """Validate every ``logging`` entry of a team/key metadata payload.""" if not metadata: return None - entries: Final = metadata.get("logging") - if not isinstance(entries, Sequence) or isinstance(entries, (str, bytes)): + entries: Final = _object_sequence(metadata.get("logging")) + if entries is None: return None entry_vars: Final = tuple(_entry_callback_vars(entry) for entry in entries) + named_vars: Final = tuple(zip((_entry_callback_name(entry) for entry in entries), entry_vars)) return next( ( error for error in ( *(_logging_entry_error(entry) for entry in entries), *(conflicting_span_scope_error(entry_vars[i], entry_vars[:i]) for i in range(len(entry_vars))), + *(_conflicting_entry_capture_error(named_vars[i], named_vars[:i]) for i in range(len(named_vars))), ) if error is not None ), @@ -211,6 +263,31 @@ def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str ) +def stored_capture_entries(logging_entries: object) -> tuple[tuple[str | None, Mapping[str, str]], ...]: + """The callback name and vars of every stored ``logging`` entry, for ``conflicting_capture_error``.""" + entries: Final = _object_sequence(logging_entries) or () + return tuple((_entry_callback_name(entry), _entry_callback_vars(entry)) for entry in entries) + + +def _object_sequence(value: object) -> Sequence[object] | None: + return value if isinstance(value, Sequence) and not isinstance(value, (str, bytes)) else None + + +def _entry_callback_name(entry: object) -> str | None: + match entry: + case {"callback_name": str() as callback_name}: + return callback_name + case _: + return None + + +def _conflicting_entry_capture_error( + entry: tuple[str | None, Mapping[str, str]], earlier: Sequence[tuple[str | None, Mapping[str, str]]] +) -> str | None: + callback_name, callback_vars = entry + return None if callback_name is None else conflicting_capture_error(callback_name, callback_vars, earlier) + + def _entry_callback_vars(entry: object) -> Mapping[str, str]: callback_vars: Final = entry.get("callback_vars") if isinstance(entry, Mapping) else None if not isinstance(callback_vars, Mapping): diff --git a/litellm/proxy/common_utils/custom_openapi_spec.py b/litellm/proxy/common_utils/custom_openapi_spec.py index c1d4442893a..45ca509b7b7 100644 --- a/litellm/proxy/common_utils/custom_openapi_spec.py +++ b/litellm/proxy/common_utils/custom_openapi_spec.py @@ -437,3 +437,41 @@ class CustomOpenAPISpec: # Add responses API request schema return CustomOpenAPISpec.add_responses_api_request_schema(with_embeddings) + + +_MAX_INLINE_DEPTH: Final = 32 + + +def _inlined(node: JsonValue, defs: Mapping[str, JsonValue], depth: int = 0) -> JsonValue: + if depth >= _MAX_INLINE_DEPTH or not isinstance(node, (list, dict)): + return node + if isinstance(node, list): + return [_inlined(item, defs, depth + 1) for item in node] + ref: Final = node.get("$ref") + if isinstance(ref, str) and ref.startswith("#/$defs/"): + target: Final = _inlined(defs[ref.removeprefix("#/$defs/")], defs, depth + 1) + siblings: Final = {key: _inlined(value, defs, depth + 1) for key, value in node.items() if key != "$ref"} + return {**target, **siblings} if isinstance(target, dict) else siblings + if "propertyName" in node and "mapping" in node: + return {"propertyName": node["propertyName"]} + return {key: _inlined(value, defs, depth + 1) for key, value in node.items() if key != "$defs"} + + +def inline_request_body(model_class: type, example: Mapping[str, object]) -> JsonObject: + """ + Build an ``openapi_extra["requestBody"]`` for a route that reads its body with ``request.body()``. + + Swagger UI resolves ``$ref`` against the whole document, so a schema placed inline in an operation + cannot keep Pydantic's ``#/$defs/...`` references. Every reference is expanded in place instead, + which also keeps the operation self-contained inside the lazy OpenAPI snapshot. + """ + schema: Final = CustomOpenAPISpec.get_pydantic_schema(model_class) + media: Final[JsonObject] = {"example": cast(JsonValue, dict(example))} # cast-ok: JSON example literal + if schema is None: + return {"required": True, "content": {"application/json": media}} + raw_defs: Final = schema.get("$defs") + defs: Final[Mapping[str, JsonValue]] = raw_defs if isinstance(raw_defs, dict) else MappingProxyType({}) + return { + "required": True, + "content": {"application/json": {"schema": _inlined(schema, defs), **media}}, + } diff --git a/litellm/proxy/common_utils/resource_ownership.py b/litellm/proxy/common_utils/resource_ownership.py index c75a5ba8e6e..985ad8a6f77 100644 --- a/litellm/proxy/common_utils/resource_ownership.py +++ b/litellm/proxy/common_utils/resource_ownership.py @@ -1,92 +1,20 @@ from typing import Final -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.litellm_core_utils.resource_ownership import ( + get_primary_resource_owner_scope, + get_resource_owner_scopes, + is_proxy_admin, + user_can_access_resource_owner, +) +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.types.proxy.auth.user_roles import LitellmUserRoles - -def is_proxy_admin(user_api_key_dict: UserAPIKeyAuth | None) -> bool: - if user_api_key_dict is None: - return False - - return ( - user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN - or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value - ) - - -def get_resource_owner_scopes( - user_api_key_dict: UserAPIKeyAuth | None, -) -> list[str]: - """ - Return ownership scopes that may access a user-created proxy resource. - - Raw user_id is included for rows created before scope prefixes existed. - Prefixes avoid collisions when falling back to team/org/key ownership - for keys that do not have a user_id. - - Identity-less callers (no user_id, team_id, org_id, api_key, or token) - return ``[]`` — they share no scope with any other caller, so access - checks against an existing owner always fail and creates that depend - on a primary scope must reject up front. Returning a shared sentinel - here would let any two identity-less callers see each other's data. - """ - if user_api_key_dict is None: - return [] - - scopes: Final[list[str]] = [] - - def _add(scope: str | None) -> None: - if scope and scope not in scopes: - scopes.append(scope) - - if user_api_key_dict.user_id: - _add(user_api_key_dict.user_id) - _add(f"user:{user_api_key_dict.user_id}") - if user_api_key_dict.team_id: - _add(f"team:{user_api_key_dict.team_id}") - if user_api_key_dict.org_id: - _add(f"org:{user_api_key_dict.org_id}") - if user_api_key_dict.api_key: - _add(f"key:{user_api_key_dict.api_key}") - if user_api_key_dict.token: - _add(f"key:{user_api_key_dict.token}") - - return scopes - - -def get_primary_resource_owner_scope( - user_api_key_dict: UserAPIKeyAuth | None, -) -> str | None: - """Return the canonical owner scope to stamp on newly-created rows. - - ``None`` for identity-less callers — callers that depend on a primary - scope to record ownership must surface that as a hard error rather - than fall back to a shared sentinel (which would collapse every - identity-less caller into the same logical owner). - """ - if user_api_key_dict is None: - return None - - if user_api_key_dict.user_id: - return user_api_key_dict.user_id - if user_api_key_dict.team_id: - return f"team:{user_api_key_dict.team_id}" - if user_api_key_dict.org_id: - return f"org:{user_api_key_dict.org_id}" - if user_api_key_dict.api_key: - return f"key:{user_api_key_dict.api_key}" - if user_api_key_dict.token: - return f"key:{user_api_key_dict.token}" - return None - - -def user_can_access_resource_owner( - owner: str | None, - user_api_key_dict: UserAPIKeyAuth | None, -) -> bool: - if user_api_key_dict is None: - return True - if is_proxy_admin(user_api_key_dict): - return True - if owner is None: - return False - return owner in get_resource_owner_scopes(user_api_key_dict) +__all__ = [ + "Final", + "LitellmUserRoles", + "UserAPIKeyAuth", + "get_primary_resource_owner_scope", + "get_resource_owner_scopes", + "is_proxy_admin", + "user_can_access_resource_owner", +] diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 4f4209208ac..c8d4768dad0 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -9,7 +9,15 @@ from pydantic import TypeAdapter, ValidationError from litellm.exceptions import BadRequestError from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body +from litellm.types.decisions import ( + DecisionsRequest, + DecisionsRequestBody, + DecisionsResponse, + OpenAIDecisionRequest, + OpenAIDecisionRequestBody, + OpenAIDecisionResponse, +) router: Final = APIRouter() _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) @@ -20,6 +28,51 @@ _OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBo _GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) _OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) +_SYSTEMONE_REQUEST_BODY: Final = inline_request_body( + DecisionsRequest, + { + "model": "jev", + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "questions": { + "is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"}, + "urgency": { + "type": "choice", + "instructions": "How urgent is this?", + "criteria": {"low": "can wait a week", "high": "needs action today"}, + }, + "frustration": { + "type": "score", + "instructions": "How frustrated is the customer?", + "criteria": ["calm", "mildly annoyed", "angry"], + }, + }, + }, +) +_DECISIONS_REQUEST_BODY: Final = inline_request_body( + OpenAIDecisionRequest, + { + "model": "luna", + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "questions": [ + {"type": "predicate", "name": "is_refund_request", "instructions": "Is the customer asking for a refund?"}, + { + "type": "choice", + "name": "urgency", + "instructions": "How urgent is this?", + "choices": [ + {"value": "low", "description": "can wait a week"}, + {"value": "high", "description": "needs action today"}, + ], + }, + { + "type": "score", + "name": "frustration", + "instructions": "How frustrated is the customer?", + "levels": [{"label": "calm"}, {"label": "mildly annoyed"}, {"label": "angry"}], + }, + ], + }, +) async def _invalid_request( @@ -121,18 +174,35 @@ async def _process_decisions( dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask named questions about a state (System One format)", + responses={200: {"model": DecisionsResponse}}, + openapi_extra={"requestBody": _SYSTEMONE_REQUEST_BODY}, ) @router.post( "/systemone", dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask named questions about a state (System One format)", + responses={200: {"model": DecisionsResponse}}, + openapi_extra={"requestBody": _SYSTEMONE_REQUEST_BODY}, ) async def systemone( request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): + """ + Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + + Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + + [Docs](https://docs.litellm.ai/docs/decisions) + """ return await _process_decisions( request=request, fastapi_response=fastapi_response, @@ -146,18 +216,35 @@ async def systemone( dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask questions about an input (OpenAI Decisions format)", + responses={200: {"model": OpenAIDecisionResponse}}, + openapi_extra={"requestBody": _DECISIONS_REQUEST_BODY}, ) @router.post( "/decisions", dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask questions about an input (OpenAI Decisions format)", + responses={200: {"model": OpenAIDecisionResponse}}, + openapi_extra={"requestBody": _DECISIONS_REQUEST_BODY}, ) async def decisions( request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): + """ + Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + + Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + model list. System One decision models accept this format too, LiteLLM translates the request and the + answers. Streaming is not supported. + + [Docs](https://docs.litellm.ai/docs/decisions) + """ return await _process_decisions( request=request, fastapi_response=fastapi_response, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0d693b85ce8..fdc2e6d94b8 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -227,6 +227,7 @@ from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS from litellm.types.services import ServiceTypes from litellm.types.utils import ( + CAPTURE_MESSAGE_CONTENT_VAR, CustomPricingLiteLLMParams, LlmProviders, ProviderSpecificHeader, @@ -975,6 +976,8 @@ def convert_key_logging_metadata_to_callback( team_callback_settings_obj.callbacks.append(data.callback_name) for var, value in data.callback_vars.items(): + if var == CAPTURE_MESSAGE_CONTENT_VAR: + continue # New Relic routing reads these from the trusted-vars overlay with no # callback-name check, so scope them to the newrelic entry: a team that # put newrelic_* under a different callback never asked for New Relic and @@ -1162,7 +1165,6 @@ def resolve_tenant_otel_destinations( span still routes to the tenant's credentials the way it did then. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled - from litellm.integrations.otel.presets.destinations import destination_for if not is_otel_v2_enabled(): return () @@ -1186,23 +1188,28 @@ def resolve_tenant_otel_destinations( for name in dict.fromkeys( callback.callback_name for callback in callbacks if callback.callback_type != "failure" ) - if ( - destination := destination_for( - name, - _tenant_otel_params( - MappingProxyType( - { - var: value - for callback in callbacks - if callback.callback_name == name - for var, value in callback.callback_vars.items() - } - ) - ), - _tenant_service_name(user_api_key_dict), + if (destination := _tenant_destination(name, callbacks, _tenant_service_name(user_api_key_dict))) is not None + ) + + +def _tenant_destination( + name: str, callbacks: Sequence[AddTeamCallback], service_name: str | None +) -> "OtelDestination | None": + from litellm.integrations.otel.model.config import parse_capture_message_content + from litellm.integrations.otel.presets.destinations import destination_for + + callback_vars: Final = MappingProxyType( + dict( + itertools.chain.from_iterable( + callback.callback_vars.items() for callback in callbacks if callback.callback_name == name ) ) - is not None + ) + return destination_for( + name, + _tenant_otel_params(callback_vars), + service_name, + parse_capture_message_content(callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR)), ) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 6e226868918..782a7e1db3c 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -8040,7 +8040,10 @@ def _validate_key_alias_format(key_alias: str | None) -> None: raise_if_unsafe_secret_name(key_alias) except ValueError: raise ProxyException( - message="Invalid key_alias", + message=( + "Invalid key_alias: must not contain control characters (such as tab or newline) " + + 'or a ".." path segment' + ), type=ProxyErrorTypes.bad_request_error, param="key_alias", code=400, diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 91fd447fb8d..3908ab3c927 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -31,8 +31,10 @@ from litellm.proxy._types import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.callback_config_validation import ( callback_config_error, + conflicting_capture_error, conflicting_span_scope_error, cross_entry_family_error, + stored_capture_entries, ) from litellm.proxy.common_utils.callback_utils import ( # noqa: F401 # legacy module exports _CALLBACK_VAR_ENCRYPTED_PREFIX, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -288,6 +290,9 @@ async def add_team_callbacks( - langfuse_host: The host for the Langfuse callback - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT) - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans + - capture_message_content: For OTel v2 callbacks, "no_content" or "span_only". + An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting. + "span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out - gcs_bucket_name: The name of the GCS bucket - gcs_path_service_account: The path to the GCS service account - langsmith_api_key: The API key for the Langsmith callback @@ -351,11 +356,19 @@ async def add_team_callbacks( # Decrypted, because the checks compare the incoming values against # the stored ones and the credentials are encrypted at rest. decrypted_logging: Final = decrypt_callback_vars(team_metadata).get("logging") + stored_capture: Final = stored_capture_entries(decrypted_logging) stored_entries: Final = decrypted_logging if isinstance(decrypted_logging, list) else () stored_entry_vars: Final = [entry.get("callback_vars") or {} for entry in stored_entries] scope_error: Final = conflicting_span_scope_error(data.callback_vars, stored_entry_vars) if scope_error is not None: raise _callback_config_error(scope_error) + capture_error: Final = conflicting_capture_error( + data.callback_name, + data.callback_vars, + stored_capture, + ) + if capture_error is not None: + raise _callback_config_error(capture_error) # One entry has to own a credential family end to end. The entries are # flattened into one dict before a request reads them, so an entry # naming only a destination would pair with a key written on another diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index f20369e279a..633e9edf71c 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -1,4 +1,22 @@ [ + { + "provider": "SCALEDOWN", + "provider_display_name": "ScaleDown", + "litellm_provider": "scaledown", + "credential_fields": [ + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "scaledown/summarize" + }, { "provider": "AIML", "provider_display_name": "AI/ML API", diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index 5a05dfe0245..a8c93eebff3 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -15,7 +15,7 @@ from litellm.types.utils import Message, ModelResponse from litellm.utils import CustomStreamWrapper if TYPE_CHECKING: - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth class _MCPCompletionKwargs(TypedDict, total=False, extra_items=object): diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index d3e3509b8f0..4dd0394005d 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -42,8 +42,8 @@ if TYPE_CHECKING: from mcp.types import Tool as MCPTool from litellm.proxy._experimental.mcp_server.ui_session_utils import GrantedToolsetIds - from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth else: MCPTool = Any diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index d2c240f8768..59b50ead83c 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -27,8 +27,8 @@ if TYPE_CHECKING: from mcp.types import Tool as MCPTool - from litellm.proxy._types import UserAPIKeyAuth from litellm.responses.mcp.litellm_proxy_mcp_handler import MCPToolResult + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth else: MCPTool = Any diff --git a/litellm/responses/mcp/request_context.py b/litellm/responses/mcp/request_context.py index 6628219e280..3254557c1f0 100644 --- a/litellm/responses/mcp/request_context.py +++ b/litellm/responses/mcp/request_context.py @@ -18,7 +18,7 @@ from pydantic import TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict if TYPE_CHECKING: - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth class _AuthCarryingMetadata(TypedDict): diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 00688782d2a..36f433a4741 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -50,7 +50,7 @@ from litellm.utils import async_post_call_success_deployment_hook if TYPE_CHECKING: from litellm.caching.caching_handler import LLMCachingHandler - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.responses.streaming_websocket import ( PresidioGuardrailCallback, ResponsesBackendWebSocket, diff --git a/litellm/router_strategy/adaptive_router/hooks.py b/litellm/router_strategy/adaptive_router/hooks.py index 0bb00a08f51..92207e07c52 100644 --- a/litellm/router_strategy/adaptive_router/hooks.py +++ b/litellm/router_strategy/adaptive_router/hooks.py @@ -28,7 +28,7 @@ from litellm.router_strategy.adaptive_router.config import ( from litellm.router_strategy.adaptive_router.signals import Turn if TYPE_CHECKING: - from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth # Identity fields hashed into a derived session key so the same conversation # from the same caller produces a stable key, while different keys/teams/users diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index b24690dbd9d..3bc90a5109a 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -15,8 +15,13 @@ MAX_DECISION_QUESTIONS: Final = 128 class NoulQuestion(LiteLLMPydanticObjectBase): type: Literal["noul"] - instructions: DecisionsJSON | None = None - criteria: NoulCriteria | None = None + instructions: Annotated[DecisionsJSON | None, Field(description="The yes/no question to ask about the state")] = ( + None + ) + criteria: Annotated[ + NoulCriteria | None, + Field(description="Optional descriptions of what makes the answer true and what makes it false"), + ] = None model_config = ConfigDict(extra="allow", frozen=True) @@ -29,16 +34,24 @@ class NoulQuestion(LiteLLMPydanticObjectBase): class ChoiceQuestion(LiteLLMPydanticObjectBase): type: Literal["choice"] - instructions: DecisionsJSON | None = None - criteria: Annotated[Mapping[str, DecisionsJSON | None], Field(min_length=1, max_length=255)] + instructions: Annotated[DecisionsJSON | None, Field(description="The question to ask about the state")] = None + criteria: Annotated[ + Mapping[str, DecisionsJSON | None], + Field(min_length=1, max_length=255, description="Candidate labels mapped to an optional description of each"), + ] model_config = ConfigDict(extra="allow", frozen=True) class ScoreQuestion(LiteLLMPydanticObjectBase): type: Literal["score"] - instructions: DecisionsJSON | None = None - criteria: Annotated[Sequence[DecisionsJSON], Field(min_length=1, max_length=10)] + instructions: Annotated[DecisionsJSON | None, Field(description="The question to ask about the state")] = None + criteria: Annotated[ + Sequence[DecisionsJSON], + Field( + min_length=1, max_length=10, description="Scale levels from lowest to highest. The score is a level index" + ), + ] model_config = ConfigDict(extra="allow", frozen=True) @@ -55,14 +68,20 @@ DecisionQuestionMap: TypeAlias = Annotated[ class DecisionsRequestBody(LiteLLMPydanticObjectBase): - state: DecisionsJSON - questions: DecisionQuestionMap + state: Annotated[ + DecisionsJSON, + Field(description="The thing being judged, such as a support ticket, a document or a chat transcript"), + ] + questions: Annotated[ + DecisionQuestionMap, + Field(description="Named noul, choice and score questions. Each key becomes a key in the answers"), + ] model_config = ConfigDict(extra="allow", frozen=True) class DecisionsRequest(DecisionsRequestBody): - model: str + model: Annotated[str, Field(description="A decision model from the proxy model_list")] @with_config(ConfigDict(extra="allow")) @@ -171,8 +190,8 @@ OpenAIDecisionInput: TypeAlias = str | Sequence[OpenAIDecisionInputMessage] class OpenAIPredicateQuestion(LiteLLMPydanticObjectBase): type: Literal["predicate"] - name: str | None = None - instructions: str + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The yes/no question to ask about the input")] model_config = ConfigDict(extra="forbid", frozen=True) @@ -192,9 +211,12 @@ def systemone_choice_key(value: str | bool) -> str: class OpenAIChoiceQuestion(LiteLLMPydanticObjectBase): type: Literal["choice"] - name: str | None = None - instructions: str - choices: Annotated[Sequence[OpenAIChoiceOption], Field(min_length=2, max_length=255)] + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The question to ask about the input")] + choices: Annotated[ + Sequence[OpenAIChoiceOption], + Field(min_length=2, max_length=255, description="The options the model picks from. Values must be unique"), + ] model_config = ConfigDict(extra="forbid", frozen=True) @@ -215,9 +237,14 @@ class OpenAIScoreLevel(LiteLLMPydanticObjectBase): class OpenAIScoreQuestion(LiteLLMPydanticObjectBase): type: Literal["score"] - name: str | None = None - instructions: str - levels: Annotated[Sequence[OpenAIScoreLevel], Field(min_length=2, max_length=10)] + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The question to ask about the input")] + levels: Annotated[ + Sequence[OpenAIScoreLevel], + Field( + min_length=2, max_length=10, description="Scale levels from lowest to highest. The score is a level index" + ), + ] model_config = ConfigDict(extra="forbid", frozen=True) @@ -229,13 +256,32 @@ OpenAIDecisionQuestion: TypeAlias = Annotated[ class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase): - input: OpenAIDecisionInput - questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=MAX_DECISION_QUESTIONS)] - safety_identifier: str | None = None + input: Annotated[ + OpenAIDecisionInput, + Field( + description="The text being judged, or user messages whose content mixes input_text and input_image parts" + ), + ] + questions: Annotated[ + Sequence[OpenAIDecisionQuestion], + Field( + min_length=1, + max_length=MAX_DECISION_QUESTIONS, + description="Predicate, choice and score questions. Answers come back in the same order", + ), + ] + safety_identifier: Annotated[ + str | None, + Field(description="A stable id for your end user. Sent to OpenAI and dropped for other providers"), + ] = None model_config = ConfigDict(extra="allow", frozen=True) +class OpenAIDecisionRequest(OpenAIDecisionRequestBody): + model: Annotated[str, Field(description="A decision model from the proxy model_list")] + + class OpenAIPredicateAnswer(LiteLLMPydanticObjectBase): type: Literal["predicate"] = "predicate" name: str | None diff --git a/litellm/types/passthrough_endpoints/managed_id_rewriter.py b/litellm/types/passthrough_endpoints/managed_id_rewriter.py index 33749cc2ab8..0603c13d905 100644 --- a/litellm/types/passthrough_endpoints/managed_id_rewriter.py +++ b/litellm/types/passthrough_endpoints/managed_id_rewriter.py @@ -24,8 +24,8 @@ from pydantic import JsonValue if TYPE_CHECKING: from litellm.models.managed_files import LiteLLM_ManagedFileTable - from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import OpenAIFileObject + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth SortOrder: TypeAlias = Literal["asc", "desc"] ResourceKind: TypeAlias = Literal["files", "batches"] diff --git a/litellm/types/proxy/auth/user_api_key_auth.py b/litellm/types/proxy/auth/user_api_key_auth.py index 3d879fdcc43..18dd4e53701 100644 --- a/litellm/types/proxy/auth/user_api_key_auth.py +++ b/litellm/types/proxy/auth/user_api_key_auth.py @@ -1,4 +1,33 @@ -from typing import Final +from collections.abc import Mapping +from typing import ( + TYPE_CHECKING, + Any, # noqa: TID251 # moved UserAPIKeyAuth fields and the Span alias are typed Any + Final, + TypeAlias, +) + +from pydantic import ConfigDict, Field, PrivateAttr, model_validator +from typing_extensions import TypeIs # noqa: TID251 # narrows the before-validator input without copying it + +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.models.team import Member +from litellm.models.verification_token import LiteLLM_VerificationToken +from litellm.types.agents import AgentCaller, AgentResponse +from litellm.types.proxy.agent_identity import ManagedAgentContext +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from litellm.types.proxy.carried_budget_state import ( + OrgBudgetSnapshot, + TeamBudgetSnapshot, + UserBudgetSnapshot, +) +from litellm.types.router import AllowedModelRegion + +if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + + Span: TypeAlias = _Span | Any +else: + Span: TypeAlias = Any def is_jwt(token: str | None) -> bool: @@ -6,3 +35,303 @@ def is_jwt(token: str | None) -> bool: return False parts: Final = token.split(".") return len(parts) == 3 + + +def hash_token(token: str) -> str: + import hashlib + + # Hash the string using SHA-256 + hashed_token: Final = hashlib.sha256(token.encode()).hexdigest() + + return hashed_token + + +_UntypedDict: TypeAlias = dict[Any, Any] + + +def _is_str_keyed_dict(value: object) -> TypeIs[dict[str, object]]: # guard-ok: pydantic input and JSON keys are str + return isinstance(value, dict) + + +class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): + """ + Combined view of litellm verification token + litellm team table (select values) + """ + + team_spend: float | None = None + team_alias: str | None = None + team_tpm_limit: int | None = None + team_rpm_limit: int | None = None + team_tpd_limit: int | None = None + team_max_budget: float | None = None + team_soft_budget: float | None = None + team_model_max_budget: dict[str, object] | None = None + team_models: list[Any] = [] + team_blocked: bool = False + soft_budget: float | None = None + team_model_aliases: _UntypedDict | None = None + team_member: Member | None = None + team_metadata: _UntypedDict | None = None + team_object_permission_id: str | None = None + + # Team Member Specific Params + team_member_spend: float | None = None + team_member_tpm_limit: int | None = None + team_member_rpm_limit: int | None = None + + # End User Params + end_user_id: str | None = None + end_user_tpm_limit: int | None = None + end_user_rpm_limit: int | None = None + end_user_tpd_limit: int | None = None + end_user_max_budget: float | None = None + end_user_model_max_budget: _UntypedDict | None = None + + # Organization Params + organization_alias: str | None = None + organization_max_budget: float | None = None + organization_tpm_limit: int | None = None + organization_rpm_limit: int | None = None + organization_metadata: _UntypedDict | None = None + + # Project Params + project_alias: str | None = None + project_metadata: _UntypedDict | None = None + + # Time stamps + last_refreshed_at: float | None = None # last time joint view was pulled from db + + def __init__(self, **kwargs: object) -> None: + # Handle litellm_budget_table_* keys (budget table overrides when key value is None or empty) + for key, value in list(kwargs.items()): + if key.startswith("litellm_budget_table_") and value is not None: + # Extract the corresponding attribute name + attr_name = key.replace("litellm_budget_table_", "") + # Use key's value from kwargs (from DB view), not class default + current = kwargs.get(attr_name) + if current is None: + current = getattr(self, attr_name, None) + # Apply budget value when key has no value, or for model_max_budget when key has empty dict + should_apply = current is None or ( + attr_name == "model_max_budget" and _is_str_keyed_dict(current) and len(current) == 0 + ) + if should_apply: + kwargs[attr_name] = value + if key == "end_user_id" and value is not None and isinstance(value, int): + kwargs[key] = str(value) + + if kwargs.get("organization_id") is not None: + kwargs["org_id"] = kwargs.pop("organization_id") + # Initialize the superclass + super().__init__(**kwargs) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # pydantic validates raw values + + +class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response object for user api key auth + """ + Return the row in the db + """ + + api_key: str | None = None + user_role: LitellmUserRoles | None = None + allowed_model_region: AllowedModelRegion | None = None + parent_otel_span: Span | None = None + rpm_limit_per_model: dict[str, int] | None = None + tpm_limit_per_model: dict[str, int] | None = None + user_tpm_limit: int | None = None + user_rpm_limit: int | None = None + user_email: str | None = None + user_spend: float | None = None + user_max_budget: float | None = None + # Values stay `object` rather than BudgetConfig: this is the raw JSON column, + # and validating it here would make one malformed row fail auth outright. + # resolve_model_budget validates the single entry a request actually needs. + user_model_max_budget: Mapping[str, object] | None = None + team_member_model_max_budget: Mapping[str, object] | None = Field(default=None, exclude=True) + request_route: str | None = None + is_session_token: bool = False + # Server-only marker set exclusively by the MCP gateway admission path + # (reload_admitted_user) for a keyless user-subject admitted via a gateway DCR session + # bearer or bridge envelope. Not a DB column and never populated from caller-controlled key + # metadata or JWT claims, so it cannot be forged to gain the team-inherited MCP grant union + # or to escape the caller-Authorization egress scrub. exclude=True keeps it out of serialization. + mcp_admitted_user_subject: bool = Field(default=False, exclude=True) + requires_fresh_policy: bool = Field(default=False, exclude=True) + mcp_explicit_grants_only: bool = Field(default=False, exclude=True) + # team_id -> that team's mcp_rpm_limit map, for a keyless admitted subject that reaches MCP + # servers through several teams at once and therefore has no single team_id for the limiter to + # key off. Server-only and stripped from validated input for the same reason as the marker + # above: a forged entry would let a caller pick which team's rpm bucket it is charged against. + mcp_source_team_rpm_limits: dict[str, dict[str, int]] | None = Field(default=None, exclude=True) + # The single MCP server_id a gateway session bearer was scoped to at authorize time (RFC 8707 + # resource), or None for an aggregate-scope session. A RESTRICTION intersected against the live + # grant resolution, never a grant. Server-only, set exclusively by the MCP gateway admission + # path via post-construction assignment and stripped from validated input like the markers + # above; a forged value could at most narrow, but the stripping keeps the field's provenance + # single-owner so its meaning stays trustworthy. + mcp_session_resource_server_id: str | None = Field(default=None, exclude=True) + mcp_toolset_id: str | None = Field(default=None, exclude=True) + authenticated_by_custom_auth: bool = Field(default=False, exclude=True) + via_virtual_key: bool = Field( + default=False, + exclude=True, + description=( + "Server-only marker set exclusively by the DB virtual-key and master-key auth paths via " + "post-construction assignment. Stripped from validated input so custom auth handlers, JWT " + "claims, or key metadata cannot forge it. Gates overwrite_user_with_key_hash stamping: only " + "a credential the proxy itself validated as a key may be forwarded as the provider-facing " + "user id." + ), + ) + invoked_agent_id: str | None = Field(default=None, exclude=True) + invoked_agent_policy: AgentResponse | None = Field(default=None, exclude=True) + agent_invocation_cost: float | None = Field(default=None, exclude=True) + billing_agent_policy: AgentResponse | None = Field(default=None, exclude=True) + _managed_delegation_verified: bool = PrivateAttr(default=False) + managed_agent_policy: AgentResponse | None = Field(default=None, exclude=True) + managed_agent_context: ManagedAgentContext | None = Field(default=None, exclude=True) + agent_caller: AgentCaller | None = Field( + default=None, + exclude=True, + description=( + "Set per request from the x-litellm-user-id / x-litellm-team-id headers an agent echoes back on " + "calls made with its own key. Every check treats it as a ceiling, so a forged value can only " + "narrow the agent's access." + ), + ) + budget_reservation: dict[str, Any] | None = Field(default=None, exclude=True) + team_budget_snapshot: TeamBudgetSnapshot | None = Field(default=None, exclude=True) + user_budget_snapshot: UserBudgetSnapshot | None = Field(default=None, exclude=True) + org_budget_snapshot: OrgBudgetSnapshot | None = Field(default=None, exclude=True) + matched_model_access_groups: list[str] | None = Field(default=None, exclude=True) + budget_throttle_pct: float | None = Field(default=None, exclude=True) + user: Any | None = None # Expanded user object when expand=user is used + created_by_user: Any | None = None # Expanded created_by user when expand=user is used + end_user_object_permission: LiteLLM_ObjectPermissionTable | None = None + # Team object_permission preloaded in auth (e.g. get_team_object) to avoid + # per-request object_permission fetches in downstream checks (vector stores, etc.) + team_object_permission: LiteLLM_ObjectPermissionTable | None = None + # Decoded upstream IdP claims (groups, roles, etc.) propagated by JWT auth machinery + # and forwarded into outbound tokens by guardrails such as MCPJWTSigner. + jwt_claims: _UntypedDict | None = None + + model_config = ConfigDict(arbitrary_types_allowed=True) + + @model_validator(mode="before") + @classmethod + def check_api_key(cls, values: object) -> object: + # If values is already an instance (not a dict), return it as-is + if not _is_str_keyed_dict(values): + return values + # mcp_admitted_user_subject is a server-only marker, set ONLY by the MCP gateway admission + # path via post-construction assignment. Strip it from any validated input (constructor + # kwargs, model_validate, a JWT/key claim splat) so it can never be forged from caller data. + values.pop("mcp_admitted_user_subject", None) + values.pop("requires_fresh_policy", None) + values.pop("mcp_explicit_grants_only", None) + values.pop("mcp_source_team_rpm_limits", None) + values.pop("mcp_session_resource_server_id", None) + values.pop("mcp_toolset_id", None) + values.pop("via_virtual_key", None) + values.pop("authenticated_by_custom_auth", None) + values.pop("agent_caller", None) + values.pop("managed_agent_context", None) + values.pop("managed_agent_policy", None) + values.pop("invoked_agent_id", None) + values.pop("invoked_agent_policy", None) + values.pop("agent_invocation_cost", None) + values.pop("billing_agent_policy", None) + api_key: Final = values.get("api_key") + if api_key is not None: + values.update({"token": cls._safe_hash_litellm_api_key(api_key)}) # pyright: ignore[reportArgumentType] # non-str fails in the hasher + if isinstance(api_key, str): + values.update({"api_key": cls._safe_hash_litellm_api_key(api_key)}) + return values + + @classmethod + def _safe_hash_litellm_api_key(cls, api_key: str) -> str: + """ + Helper to ensure all logged keys are hashed + Covers: + 1. Regular API keys from LiteLLM DB + 2. JWT tokens used for connecting to LiteLLM API + """ + normalized = api_key + if normalized[:7].lower() == "bearer ": + normalized = normalized[7:] + if normalized.startswith("sk-"): + return hash_token(normalized) + if is_jwt(token=normalized): + return f"hashed-jwt-{hash_token(token=normalized)}" + return normalized + + @classmethod + def get_litellm_internal_health_check_user_api_key_auth(cls) -> "UserAPIKeyAuth": + """ + Returns a `UserAPIKeyAuth` object for the litellm internal health check service account. + + This is used to track number of requests/spend for health check calls. + """ + from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME + + return cls( + api_key=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + team_id=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + key_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + team_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + ) + + @classmethod + def get_litellm_cli_user_api_key_auth(cls) -> "UserAPIKeyAuth": + """ + Returns a `UserAPIKeyAuth` object for the litellm internal health check service account. + + This is used to track number of requests/spend for health check calls. + """ + from litellm.constants import LITTELM_CLI_SERVICE_ACCOUNT_NAME + + return cls( + api_key=LITTELM_CLI_SERVICE_ACCOUNT_NAME, + team_id=LITTELM_CLI_SERVICE_ACCOUNT_NAME, + key_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, + team_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, + ) + + @classmethod + def get_litellm_internal_jobs_user_api_key_auth(cls) -> "UserAPIKeyAuth": + """ + Returns a `UserAPIKeyAuth` object for internal LiteLLM jobs like key rotation. + + This is used to track actions performed by automated system jobs. + """ + from litellm.constants import LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME + + return cls( + api_key=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + team_id="system", + key_alias=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + team_alias="system", + user_id="system", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + @property + def is_team_service_account(self) -> bool: + return ( + self.user_id is None + and self.team_id is not None + and bool(self.metadata) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped base field + and self.metadata.get("service_account_id") is not None # pyright: ignore[reportUnknownMemberType] # untyped base field + ) + + +def user_api_key_has_admin_view(user_api_key_dict: UserAPIKeyAuth) -> bool: + """Return True if the caller's role grants unscoped read access to all + tenant resources (managed files, batches, vector stores, spend rows, etc). + + Lives on _types.py so leaf modules (e.g. litellm.llms.base_llm.managed_resources) + can use it without pulling in litellm.proxy.utils via management_endpoints. + """ + return user_api_key_dict.user_role in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ) diff --git a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py index 1fd3e7dca53..febd4485d10 100644 --- a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py @@ -4,14 +4,14 @@ from typing import Any, Final, Literal from pydantic import ConfigDict, Field, field_validator from typing_extensions import ReadOnly, TypedDict -from litellm.proxy._types import ( +from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse +from litellm.types.proxy.management_endpoints.user_requests import ( LiteLLM_UserTableWithKeyCount, NewUserRequest, UpdateUserRequest, UpdateUserRequestNoUserIDorEmail, ) -from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse MAX_BULK_DELETE_USERS: Final = 500 diff --git a/litellm/types/proxy/management_endpoints/key_management_endpoints.py b/litellm/types/proxy/management_endpoints/key_management_endpoints.py index 05b63100ea4..f4348391710 100644 --- a/litellm/types/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/key_management_endpoints.py @@ -5,14 +5,14 @@ from pydantic import ConfigDict, model_validator from typing_extensions import ReadOnly, TypedDict from litellm.models.verification_token import LiteLLM_VerificationToken -from litellm.proxy._types import ( +from litellm.types.llms.base import LiteLLMBaseModel, LiteLLMPydanticObjectBase +from litellm.types.proxy.management_endpoints.internal_user_endpoints import InsensitiveContains +from litellm.types.proxy.management_endpoints.key_requests import ( GenerateKeyRequest, - LiteLLM_ObjectPermissionBase, RegenerateKeyRequest, UpdateKeyRequest, ) -from litellm.types.llms.base import LiteLLMBaseModel, LiteLLMPydanticObjectBase -from litellm.types.proxy.management_endpoints.internal_user_endpoints import InsensitiveContains +from litellm.types.proxy.management_endpoints.request_base import LiteLLM_ObjectPermissionBase class KeyTokenWhere(TypedDict): diff --git a/litellm/types/proxy/management_endpoints/key_requests.py b/litellm/types/proxy/management_endpoints/key_requests.py new file mode 100644 index 00000000000..baa02b68d98 --- /dev/null +++ b/litellm/types/proxy/management_endpoints/key_requests.py @@ -0,0 +1,138 @@ +import enum +from collections.abc import Mapping +from datetime import datetime +from types import MappingProxyType +from typing import Any, Literal # noqa: TID251 # allowed_routes and metadata stay Any for existing callers + +from pydantic import Field, PositiveInt, field_validator, model_validator +from typing_extensions import TypeIs # noqa: TID251 # narrows the raw payload a before-validator receives + +from litellm.types.proxy.management_endpoints.request_base import GenerateRequestBase +from litellm.types.router import UpdateRouterConfig +from litellm.types.utils import LiteLLMPydanticObjectBase + + +class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase): + index_name: str + index_permissions: list[Literal["read", "write"]] + + +class KeyRequestBase(GenerateRequestBase): + key: str | None = None + tpd_limit: int | None = None + default_estimated_output_tokens: PositiveInt | None = None + default_estimated_output_tokens_per_model: Mapping[str, PositiveInt] | None = None + budget_id: str | None = None + end_user_budget_id: str | None = None + tags: list[str] | None = None + disable_global_guardrails: bool | None = None + enable_prompt_caching: bool | None = None + throttle_on_budget_exceeded: bool | None = None + enforced_params: list[str] | None = None + allowed_routes: list[Any] | None = [] + allowed_passthrough_routes: list[object] | None = None + denied_passthrough_routes: list[str] | None = None + allowed_vector_store_indexes: list[AllowedVectorStoreIndexItem] | None = None + rpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( + None # raise an error if 'guaranteed_throughput' is set and we're overallocating rpm + ) + tpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( + None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm + ) + router_settings: UpdateRouterConfig | None = None + access_group_ids: list[str] | None = None + + +class LiteLLMKeyType(str, enum.Enum): + """ + Enum for key types that determine what routes a key can access + """ + + LLM_API = "llm_api" # Can call LLM API routes (chat/completions, embeddings, etc.) + MANAGEMENT = "management" # Can call management routes (user/team/key management) + READ_ONLY = "read_only" # Can only call info/read routes + DEFAULT = "default" # Uses default allowed routes + + +class GenerateKeyRequest(KeyRequestBase): + soft_budget: float | None = None + send_invite_email: bool | None = None + key_type: LiteLLMKeyType | None = Field( + default=LiteLLMKeyType.DEFAULT, + description="Type of key that determines default allowed routes.", + ) + auto_rotate: bool | None = Field(default=False, description="Whether this key should be automatically rotated") + rotation_interval: str | None = Field( + default=None, + description="How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True", + ) + organization_id: str | None = None + project_id: str | None = None + + @field_validator("team_id", "organization_id", "project_id", mode="before") + @classmethod + def treat_cleared_id_as_unset(cls, v: object) -> object: + if v == "": + return None + return v + + +def _is_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: raw before-validator payload + return isinstance(value, Mapping) + + +class UpdateKeyRequest(KeyRequestBase): + # Note: the defaults of all Params here MUST BE NONE + # else they will get overwritten + duration: str | None = None + spend: float | None = None + soft_budget: float | None = None + metadata: dict[Any, Any] | None = None + temp_budget_increase: float | None = None + temp_budget_expiry: datetime | None = None + auto_rotate: bool | None = None + rotation_interval: str | None = None + organization_id: str | None = None + + project_id: str | None = Field( + default=None, + description="Omit to retain the project, or send null to detach. Assigning a different project is not supported.", + ) + + @model_validator(mode="before") + @classmethod + def drop_blank_team_id(cls, values: object) -> object: + if _is_mapping(values) and values.get("team_id") == "": + return MappingProxyType({k: v for k, v in values.items() if k != "team_id"}) + return values + + @field_validator("organization_id", mode="before") + @classmethod + def treat_cleared_organization_id_as_unset(cls, v: object) -> object: + if v == "": + return None + return v + + @model_validator(mode="after") + def validate_temp_budget(self) -> "UpdateKeyRequest": + if self.temp_budget_increase is not None or self.temp_budget_expiry is not None: + if self.temp_budget_increase is None or self.temp_budget_expiry is None: + raise ValueError("temp_budget_increase and temp_budget_expiry must be set together") + return self + + @model_validator(mode="after") + def validate_key_identifier(self) -> "UpdateKeyRequest": + if self.key is None and self.key_alias is None: + raise ValueError("either key or key_alias must be provided") + return self + + +class RegenerateKeyRequest(GenerateKeyRequest): + # This needs to be different from UpdateKeyRequest, because "key" is optional for this + key: str | None = None + new_key: str | None = None + duration: str | None = None + spend: float | None = None + metadata: dict[Any, Any] | None = None + new_master_key: str | None = None + grace_period: str | None = None # Duration to keep old key valid (e.g. "24h", "2d"); None = immediate revoke diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 796bcae37bc..065205dcad8 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,16 +1,15 @@ +from collections.abc import Mapping from typing import Any, Final, Literal from pydantic import ConfigDict, Field, field_validator, model_validator from litellm.models.team import LiteLLM_DeletedTeamTable, LiteLLM_TeamTable, Member from litellm.models.team_membership import LiteLLM_TeamMembership -from litellm.proxy._types import ( - KeyManagementRoutes, - MemberDeleteRequest, -) from litellm.proxy.common_utils.timezone_utils import budget_duration_error from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.proxy.auth.user_roles import KeyManagementRoutes from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse +from litellm.types.utils import LiteLLMPydanticObjectBase TeamIdSearchMatch = Literal["exact", "prefix"] @@ -19,6 +18,18 @@ MAX_BULK_TEAM_MEMBER_DELETES: Final = 500 MAX_BULK_TEAM_MEMBER_BUDGET_UPDATES: Final = 500 +class MemberDeleteRequest(LiteLLMPydanticObjectBase): + user_id: str | None = None + user_email: str | None = None + + @model_validator(mode="before") + @classmethod + def check_user_info(cls, values: Mapping[str, object]) -> Mapping[str, object]: + if values.get("user_id") is None and values.get("user_email") is None: + raise ValueError("Either user id or user email must be provided") + return values + + class GetTeamMemberPermissionsRequest(LiteLLMBaseModel): """Request to get the team member permissions for a team""" diff --git a/litellm/types/proxy/management_endpoints/user_requests.py b/litellm/types/proxy/management_endpoints/user_requests.py new file mode 100644 index 00000000000..49dda5995bc --- /dev/null +++ b/litellm/types/proxy/management_endpoints/user_requests.py @@ -0,0 +1,82 @@ +from collections.abc import Mapping +from typing import Literal + +from pydantic import Field, field_validator, model_validator + +from litellm.models.user import LiteLLM_UserTable +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from litellm.types.proxy.management_endpoints.request_base import GenerateRequestBase +from litellm.types.utils import LiteLLMPydanticObjectBase + + +class NewUserRequestTeam(LiteLLMPydanticObjectBase): + team_id: str + max_budget_in_team: float | None = None + user_role: Literal["user", "admin"] = "user" + + +class NewUserRequest(GenerateRequestBase): + max_budget: float | None = None + user_email: str | None = None + user_alias: str | None = None + user_role: ( + Literal[ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ] + | None + ) = None + teams: list[str] | list[NewUserRequestTeam] | None = None + auto_create_key: bool = True # flag used for returning a key as part of the /user/new response + send_invite_email: bool | None = None + sso_user_id: str | None = None + organizations: list[str] | None = None + password: str | None = None + + @field_validator("password") + @classmethod + def password_not_supported(cls, value: str | None) -> str | None: + if value is not None: + raise ValueError( + "password cannot be set via /user/new. Users set their own password through an " + "invitation link (POST /invitation/new)." + ) + return value + + +class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase): # shared with BulkUpdateUserRequest + # repr=False keeps the plaintext out of management-endpoint alerts, which str() the request model + password: str | None = Field(default=None, repr=False) + spend: float | None = None + metadata: dict[object, object] | None = None + user_alias: str | None = None + user_role: ( + Literal[ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ] + | None + ) = None + max_budget: float | None = None + + +class UpdateUserRequest(UpdateUserRequestNoUserIDorEmail): + # Note: the defaults of all Params here MUST BE NONE + # else they will get overwritten + user_id: str | None = None + user_email: str | None = None + + @model_validator(mode="before") + @classmethod + def check_user_info(cls, values: Mapping[str, object]) -> Mapping[str, object]: + if values.get("user_id") is None and values.get("user_email") is None: + raise ValueError("Either user id or user email must be provided") + return values + + +class LiteLLM_UserTableWithKeyCount(LiteLLM_UserTable): + key_count: int = 0 diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4a962e5b3bd..13fd86b74a7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3726,6 +3726,28 @@ ArizeOtlpProtocol = Literal["grpc", "http/protobuf"] ARIZE_OTLP_PROTOCOLS: Final[frozenset[str]] = frozenset(get_args(ArizeOtlpProtocol)) +CAPTURE_MESSAGE_CONTENT_VAR: Final = "capture_message_content" + + +class CaptureMessageContent(str): + NO_CONTENT = "no_content" + SPAN_ONLY = "span_only" + EVENT_ONLY = "event_only" + SPAN_AND_EVENT = "span_and_event" + + +# The values a team or key destination may set. OTel v2 writes content only to span attributes, +# so the event modes stay global-only settings +CAPTURE_MESSAGE_CONTENT_VALUES: Final[frozenset[str]] = frozenset( + {CaptureMessageContent.NO_CONTENT, CaptureMessageContent.SPAN_ONLY} +) + + +def captures_span_content(mode: str | None) -> bool: + """Whether a capture mode puts prompt and response content on spans""" + return mode in (CaptureMessageContent.SPAN_ONLY, CaptureMessageContent.SPAN_AND_EVENT) + + class StandardCallbackDynamicParams(TypedDict, total=False): # Langfuse dynamic params langfuse_public_key: str | None @@ -4202,6 +4224,7 @@ class LlmProviders(str, Enum): FRIENDLIAI = "friendliai" FEATHERLESS_AI = "featherless_ai" WATSONX = "watsonx" + SCALEDOWN = "scaledown" WATSONX_TEXT = "watsonx_text" TRITON = "triton" PREDIBASE = "predibase" diff --git a/litellm/utils.py b/litellm/utils.py index e7a171f56c6..6774771dbf9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8655,6 +8655,7 @@ class ProviderConfigManager: False, ), LlmProviders.FEATHERLESS_AI: (lambda: litellm.FeatherlessAIConfig(), False), + LlmProviders.SCALEDOWN: (lambda: litellm.ScaleDownChatConfig(), False), LlmProviders.NOVITA: (lambda: litellm.NovitaConfig(), False), LlmProviders.NEBIUS: (lambda: litellm.NebiusConfig(), False), LlmProviders.WANDB: (lambda: litellm.WandbConfig(), False), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cb910ae12ea..bf7f1db7a31 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -66876,7 +66876,7 @@ "supports_audio_input": true }, "azure/gpt-realtime-translate": { - "deprecation_date": "2027-05-06", + "deprecation_date": "2027-11-09", "input_cost_per_second": 0.000566666666667, "litellm_provider": "azure", "max_input_tokens": 32000, @@ -71842,5 +71842,51 @@ "image" ], "supports_embedding_image_input": true + }, + "scaledown/extract": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/summarize": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": true, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/compress": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": true, + "supports_function_calling": false, + "supports_vision": false + }, + "scaledown/classify": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/decisions": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false } } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 33ac4e4e60b..ea617cf8462 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2349,6 +2349,24 @@ "search": true } }, + "scaledown": { + "display_name": "ScaleDown (`scaledown`)", + "url": "https://docs.litellm.ai/docs/providers/scaledown", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "sambanova": { "display_name": "Sambanova (`sambanova`)", "url": "https://docs.litellm.ai/docs/providers/sambanova", diff --git a/scripts/check_openapi_docs.py b/scripts/check_openapi_docs.py new file mode 100755 index 00000000000..26bb034bfa6 --- /dev/null +++ b/scripts/check_openapi_docs.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""List the POST operations in the proxy's OpenAPI document that Swagger UI cannot explain. + +Every finding is one line, `` ``, so scripts/openapi_docs_gate.py can +count them per rule the same delta-vs-base way as the other lint gates. The rules: + +OA001 no request body schema (the handler reads ``request.body()`` and never told FastAPI + what is in it, see ``inline_request_body`` in custom_openapi_spec.py for the fix) +OA002 no description (the handler has no docstring and the route no ``description=``) +OA003 the 200 response has no schema (no ``response_model`` and no ``responses={200: ...}``) +""" + +from __future__ import annotations + +import json +import sys +from collections.abc import Iterator, Mapping +from typing import Final, NamedTuple + +GATED_METHODS: Final = frozenset({"post"}) + + +class Violation(NamedTuple): + method: str + path: str + code: str + + +def _as_mapping(node: object) -> Mapping[str, object]: + return node if isinstance(node, Mapping) else {} + + +def _json_schema(container: object) -> object: + content: Final = _as_mapping(_as_mapping(container).get("content")) + return _as_mapping(content.get("application/json")).get("schema") + + +def _operation_codes(operation: Mapping[str, object]) -> Iterator[str]: + if not _json_schema(operation.get("requestBody")): + yield "OA001" + description: Final = operation.get("description") + if not isinstance(description, str) or not description.strip(): + yield "OA002" + if not _json_schema(_as_mapping(operation.get("responses")).get("200")): + yield "OA003" + + +def violations(spec: Mapping[str, object]) -> tuple[Violation, ...]: + paths: Final = _as_mapping(spec.get("paths")) + return tuple( + Violation(method.upper(), path, code) + for path, operations in sorted(paths.items()) + for method, operation in sorted(_as_mapping(operations).items()) + if method in GATED_METHODS + for code in _operation_codes(_as_mapping(operation)) + ) # comprehension-ok: one pass over path, method and code is the whole scan + + +def proxy_spec() -> Mapping[str, object]: + from litellm.proxy.proxy_server import app + + return _as_mapping(app.openapi()) + + +def main(argv: list[str] = sys.argv[1:]) -> int: + spec: Final = json.loads(open(argv[0]).read()) if argv else proxy_spec() + for violation in violations(spec): + print(f"{violation.method} {violation.path} {violation.code}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/openapi_docs_gate.py b/scripts/openapi_docs_gate.py new file mode 100755 index 00000000000..a2a8fad78e9 --- /dev/null +++ b/scripts/openapi_docs_gate.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Delta-vs-base gate for the OA* rules in scripts/check_openapi_docs.py. + +Sibling of scripts/test_quality_gate.py, pointed at the proxy's OpenAPI document +instead of a source tree. Each rule is counted over every POST operation the proxy +serves at HEAD and at the merge-base with the branch this change merges into, and the +gate fails only when a rule grew past the merge-base count, so a new endpoint has to +ship with a request schema, a description and a 200 schema while the undocumented +routes that already exist are only ever allowed to shrink. + +The merge-base counts come from scripts/lint_base_counts.py: the disk cache, then the +CI artifact published for that commit, then a pass of the current checker over a +detached worktree at the merge-base with that tree first on PYTHONPATH. +""" + +from __future__ import annotations + +import argparse +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +from collections import Counter +from collections.abc import Mapping, Sequence +from pathlib import Path +from types import FrameType, MappingProxyType +from typing import Final, NamedTuple + +from lint_base_counts import ( + Checker, + base_counts_cached, + emit_counts, + evaluate, + head_sha, + resolve_base_point, + sha256_of, +) + +REPO_ROOT: Final = Path(__file__).resolve().parent.parent +CHECKER: Final = REPO_ROOT / "scripts" / "check_openapi_docs.py" +TARGET: Final = "litellm" +TERMINATION_SIGNALS: Final = (signal.SIGTERM, signal.SIGHUP) + +_LINE: Final = re.compile(r"^(?P[A-Z]+) (?P\S+) (?POA\d+)$") +_ADDED_LINE: Final = re.compile(r"^\+(?!\+\+)(?P.*)$", re.MULTILINE) + + +class Violation(NamedTuple): + method: str + path: str + code: str + + +def _run(cmd: Sequence[str], cwd: Path = REPO_ROOT, env: Mapping[str, str] | None = None) -> str: + proc: Final = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + if proc.returncode != 0: + sys.stderr.write(proc.stderr) + raise SystemExit(f"{cmd[0]} exited {proc.returncode}") + return proc.stdout + + +def checker_identity(checker: Path = CHECKER) -> Checker: + return Checker("openapi-docs", (sha256_of(checker),)) + + +def parse_violations(out: str) -> tuple[Violation, ...]: + return tuple( + Violation(match.group("method"), match.group("path"), match.group("code")) + for line in out.splitlines() + if (match := _LINE.match(line)) is not None + ) + + +def _check(root: Path, checker: Path) -> tuple[Violation, ...]: + resolved: Final = root.resolve() + env: Final = {**os.environ, "PYTHONPATH": os.pathsep.join((str(resolved), os.environ.get("PYTHONPATH", "")))} + return parse_violations(_run([sys.executable, str(checker)], cwd=resolved, env=env)) + + +def head_violations() -> tuple[Violation, ...]: + return _check(REPO_ROOT, CHECKER) + + +def count_by_rule(violations: Sequence[Violation]) -> Mapping[str, int]: + return MappingProxyType(dict(Counter(v.code for v in violations))) + + +def _exit_on_termination(signum: int, _frame: FrameType | None) -> None: + raise SystemExit(128 + signum) + + +def _install_termination_handlers() -> None: + for termination in TERMINATION_SIGNALS: + if signal.getsignal(termination) == signal.SIG_DFL: + signal.signal(termination, _exit_on_termination) + + +def base_counts(ref: str, repo_root: Path = REPO_ROOT, checker: Path = CHECKER) -> Mapping[str, int]: + """Rule counts at `ref`, measured with the *current* checker over the base tree.""" + _install_termination_handlers() + parent: Final = Path(tempfile.mkdtemp(prefix="oa_base_")) + worktree: Final = parent / "wt" + try: + _run(["git", "worktree", "add", "--detach", str(worktree), ref], cwd=repo_root) + (worktree / "scripts").mkdir(parents=True, exist_ok=True) + base_checker: Final = worktree / "scripts" / "check_openapi_docs.py" + shutil.copy(checker, base_checker) + return count_by_rule(_check(worktree, base_checker)) + finally: + # Teardown must never raise, or it masks the real error when the body failed. + subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + cwd=repo_root, capture_output=True, text=True, + ) + shutil.rmtree(parent, ignore_errors=True) + + +def added_text(diff_text: str) -> str: + return "\n".join(match.group("text") for match in _ADDED_LINE.finditer(diff_text)) + + +def introduced(violations: Sequence[Violation], added: str) -> tuple[Violation, ...]: + """The violations whose route path is quoted on a line this change added. A new + route is registered with its path as a string literal, so this names the endpoint + the change brought in without needing the base document's own violation list.""" + return tuple(v for v in violations if f'"{v.path}"' in added or f"'{v.path}'" in added) + + +def cmd_check(base: str) -> None: + head: Final = head_violations() + base_point: Final = resolve_base_point(base) + breaches: Final = evaluate(count_by_rule(head), base_counts_cached(checker_identity(), base_point, base_counts)) + if not breaches: + print(f"OK: no OA rule grew past its merge-base count (base {base})") + return + diff: Final = _run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET]) + new: Final = introduced(head, added_text(diff)) + print(f"FAIL: OA-rule totals grew past their merge-base count (base {base}):") + for breach in breaches: + print(f" {breach.rule}: total {breach.total} over ceiling {breach.ceiling} (this change added {breach.added})") + for violation in sorted(v for v in new if v.code == breach.rule): + print(f" {violation.method} {violation.path}") + print( + "Give every new POST route a request body schema (a typed body parameter, or " + "`openapi_extra={'requestBody': inline_request_body(Model, example)}` when the handler reads " + "`request.body()`), a docstring, and a 200 schema (`response_model=` or `responses={200: {'model': ...}}`), " + "or document an equal number of existing routes; the ceiling is the merge-base count. Run " + "`python scripts/check_openapi_docs.py` to see every finding." + ) + raise SystemExit(1) + + +def main() -> None: + parser: Final = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", help="Comparison ref (default: origin's current default branch)") + parser.add_argument( + "--emit-counts-dir", + type=Path, + help="Write HEAD's per-rule counts to this directory as a base-counts artifact instead of gating", + ) + args: Final = parser.parse_args() + from default_branch import resolve_base_ref + from gate_slot_lock import held_slot + + if args.emit_counts_dir is not None: + with held_slot(): + emit_counts(checker_identity(), count_by_rule(head_violations()), args.emit_counts_dir, head_sha()) + return + base_ref: Final = resolve_base_ref(args.base, REPO_ROOT) + with held_slot(): + cmd_check(base_ref) + + +if __name__ == "__main__": + main() diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 7166891bb1a..e1d58ee73d2 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -58,6 +58,7 @@ IGNORE_FUNCTIONS = [ "sanitize_oci_schema", # OCI: bounded by JSON-schema tree depth. "_freeze_for_dedupe", # OTEL: max depth set (default 16, _FREEZE_MAX_DEPTH); fails closed by returning repr(value) at the cap. "apply_json_merge_patch", # max depth set (_MAX_MERGE_DEPTH=64); fails closed by raising ValueError at the cap. + "_inlined", # max depth set (_MAX_INLINE_DEPTH=32); passes the schema node through untouched at the cap. Walks a Pydantic JSON schema once at import time. "_filter_argument_value", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by blocking the tool call at the cap. "_redact_scanned_content", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by returning "[REDACTED]" at the cap. "replace_ciphertexts", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); walks stored JSON, which has no cycles, and leaves values below the cap untouched. @@ -67,6 +68,7 @@ IGNORE_FUNCTIONS = [ "strict_json_schema", # harness: max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by raising ValueError at the cap. "toml_value", # harness/codex: max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by raising OptionsMismatch at the cap. "with_json_string_leaves", # transitively bounded: only runs on a tree json_string_leaves already walked under the cap. + "_clean_extraction", # ScaleDown: traverses the entity map built under MAX_SCHEMA_DEPTH=32 and MAX_ENTITIES=1000. "json_unrewritable_labels", # max depth set (MAX_STRUCTURED_CONTENT_SCAN_DEPTH); returns the None sentinel at the cap so the caller blocks. "_flatten_form_field", # bounded by the nesting depth of the already-parsed request body (a finite JSON tree, no cycles possible). "_flatten_form_data_field", # bounded by the nesting depth of the already-parsed request body (a finite JSON tree, no cycles possible). diff --git a/tests/integration/_support/mcp.py b/tests/integration/_support/mcp.py index 68bffc85f40..c51ecfc96b1 100644 --- a/tests/integration/_support/mcp.py +++ b/tests/integration/_support/mcp.py @@ -4,7 +4,7 @@ import os import queue import sys import time -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Generator, Iterator, Mapping from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path @@ -383,6 +383,55 @@ def peer_of(kind: PeerKind, *, rich: bool = False) -> Iterator[McpPeer]: yield candidate +@contextmanager +def stateful_mcp_peer() -> Generator[McpPeer]: + service: Final = MCPServer("integration-stateful") + selected: Final[dict[str, str]] = {} # mutable-ok: per-session state the upstream keeps across tool calls + + def upstream_session(ctx: Context) -> str: + assert ctx.headers is not None, "stateful upstream requires HTTP request headers" + return ctx.headers["mcp-session-id"] + + @service.tool() + def select_project(name: str, ctx: Context) -> str: + selected[upstream_session(ctx)] = name + return f"selected {name}" + + @service.tool() + def create_feature(title: str, ctx: Context) -> str: + project: Final = selected.get(upstream_session(ctx)) + if project is None: + raise ValueError("no project selected in this session") + return f"{project}/{title}" + + app: Final = service.streamable_http_app( + stateless_http=False, + json_response=True, + transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False), + ) + observed: Final[queue.Queue[dict[str, object]]] = queue.Queue() + + async def capture(scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] == "http" and scope["method"] == "GET": + await Response(status_code=405)(scope, receive, send) + return + if scope["type"] != "http" or scope["method"] != "POST": + await app(scope, receive, send) + return + body: Final = await StarletteRequest(scope, receive).body() + observed.put({"body": json.loads(body) if body else None, "headers": dict(scope["headers"])}) + message: Final[Message] = {"type": "http.request", "body": body, "more_body": False} + pending: Final = iter((message,)) + + async def replay() -> Message: + return next(pending, {"type": "http.disconnect"}) + + await app(scope, replay, send) + + with asgi_server(capture) as url: + yield McpPeer(url + "/mcp", observed) + + def register_mcp(scenario: Scenario, peer: McpPeer, alias: str, **fields: object) -> str: response: Final = scenario.gateway.request( "POST", "/v1/mcp/server", {"server_name": alias, "alias": alias, **peer.registration(), **fields} diff --git a/tests/integration/_support/otlp_sink.py b/tests/integration/_support/otlp_sink.py index eeabe9d886f..a4d2ea30959 100644 --- a/tests/integration/_support/otlp_sink.py +++ b/tests/integration/_support/otlp_sink.py @@ -186,6 +186,7 @@ class _Handler(BaseHTTPRequestHandler): "count": len(recorded), "host": self.headers.get("host", ""), "headers": dict(self.headers), + "span_ids": [span["span_id"] for span in recorded], } ) self._send_json({"recorded": len(recorded)}, status=self.state.status) @@ -466,9 +467,7 @@ def _await_sink(url: str) -> None: def owned_grpc_sink(directory: Path) -> Iterator[GrpcSink]: http_port: Final = _free_port() grpc_port: Final = _free_port() - with _spawn_sink( - directory, "otlp-grpc-sink.log", ["--port", str(http_port), "--grpc-port", str(grpc_port)] - ): + with _spawn_sink(directory, "otlp-grpc-sink.log", ["--port", str(http_port), "--grpc-port", str(grpc_port)]): control_url: Final = f"http://127.0.0.1:{http_port}" _await_sink(control_url) yield GrpcSink(url=f"http://127.0.0.1:{grpc_port}", control_url=control_url) diff --git a/tests/integration/mcp/test_mcp_lifecycle.py b/tests/integration/mcp/test_mcp_lifecycle.py index 464e3d1e9a8..be5f5d3a651 100644 --- a/tests/integration/mcp/test_mcp_lifecycle.py +++ b/tests/integration/mcp/test_mcp_lifecycle.py @@ -1,3 +1,4 @@ +import asyncio import functools import json import uuid @@ -5,13 +6,18 @@ from collections.abc import Mapping from contextlib import ExitStack from hashlib import sha256 from pathlib import Path -from typing import Final +from typing import Final, cast +import httpx +import httpx2 import pytest import yaml from hypothesis import settings from hypothesis import strategies as st from hypothesis.stateful import RuleBasedStateMachine, invariant, rule, run_state_machine_as_test +from mcp import ClientSession +from mcp.client.streamable_http import streamable_http_client +from mcp.types import CallToolResult, TextContent from integration._support.client import Gateway, eventually from integration._support.database import read_rows from integration._support.generation import LIFECYCLE_SETTINGS, bounded_http_requests @@ -25,6 +31,7 @@ from integration._support.mcp import ( register_mcp, scripted_peer, text_result, + stateful_mcp_peer, tool_calls, tool_names, ) @@ -466,6 +473,42 @@ def test_key_grant_added_by_key_update_is_visible_to_mcp_tool_listing_before_the assert set(seen.tools) == {f"{alias}-add", f"{alias}-multiply", f"{alias}-fail"}, seen.raw +async def _select_then_create(url: str, key: str) -> tuple[CallToolResult, CallToolResult]: + async with httpx2.AsyncClient(headers={"x-litellm-api-key": key}, timeout=30, trust_env=False) as http: + async with streamable_http_client(url, http_client=http) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + names: Final = {tool.name.rsplit("-", 1)[-1]: tool.name for tool in (await session.list_tools()).tools} + selected: Final = await session.call_tool(names["select_project"], {"name": "alpha"}) + created: Final = await session.call_tool(names["create_feature"], {"title": "login"}) + return selected, created + + +def _upstream_session_ids(observed: tuple[dict[str, object], ...]) -> set[str | None]: + headers: Final = tuple(cast(dict[bytes, bytes], item["headers"]) for item in observed) + return {raw.decode() if isinstance(raw := item.get(b"mcp-session-id"), bytes) else None for item in headers} + + +def test_stateful_upstream_keeps_selection_across_tool_calls_through_gateway(gateway: Gateway) -> None: + with stateful_mcp_peer() as peer, gateway.scenario() as scenario: + alias: Final = "integration" + uuid.uuid4().hex + identity: Final = register_mcp(scenario, peer, alias) + key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) + peer.drain() + selected, created = asyncio.run(_select_then_create(f"{gateway.client.base_url}/{alias}/mcp", key)) + observed: Final = peer.drain() + for session_id in _upstream_session_ids(observed): + if session_id is not None: + httpx.delete(peer.url, headers={"mcp-session-id": session_id}, trust_env=False) + call_sessions: Final = _upstream_session_ids( + tuple(item for item in observed if cast(dict[str, object], item["body"]).get("method") == "tools/call") + ) + assert selected.is_error is False, selected.model_dump_json() + assert created.is_error is False, created.model_dump_json() + assert isinstance(created.content[0], TextContent) and created.content[0].text == "alpha/login" + assert len(call_sessions) == 1 and None not in call_sessions, call_sessions + + def _update_tool_permissions( gateway: Gateway, key: str, identity: str, permissions: dict[str, list[str]] | None ) -> None: diff --git a/tests/integration/observability/test_otel_team_capture_message_content.py b/tests/integration/observability/test_otel_team_capture_message_content.py new file mode 100644 index 00000000000..84da9a40486 --- /dev/null +++ b/tests/integration/observability/test_otel_team_capture_message_content.py @@ -0,0 +1,720 @@ +import json +import re +import uuid +from collections.abc import Generator, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal +from urllib.parse import urlparse + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, object_value +from integration._support.otlp_sink import ( + ConnectSink, + Span, + SpanSinks, + configure_sink, + owned_connect_sink, + recorded_requests, + recorded_spans, +) +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +MARKER: Final = re.compile(rb"cmc-(?:tool-|fail-)?[0-9a-f]{32}") +JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +SERVER: Final = 2 +CALL_ID: Final = "litellm.call_id" +OPERATION: Final = "gen_ai.operation.name" +INPUT_TOKENS: Final = "gen_ai.usage.input_tokens" +Endpoint = Literal["chat", "responses", "messages"] +Capture = Literal["no_content", "span_only"] + + +def _marker(kind: str = "") -> str: + return f"cmc-{kind}{uuid.uuid4().hex}" + + +def _usage() -> dict[str, JsonValue]: + return {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9} + + +def _chat_reply(marker: str, stream: bool) -> Reply: + identity: Final = f"chatcmpl-{uuid.uuid4().hex}" + if marker.startswith("cmc-tool-"): + call: Final[dict[str, JsonValue]] = { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": json.dumps({"query": marker})}, + } + message: Final[dict[str, JsonValue]] = {"role": "assistant", "content": None, "tool_calls": [call]} + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": message, "finish_reason": "tool_calls"}], + "usage": _usage(), + } + ).encode() + ) + if not stream: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": marker}, "finish_reason": "stop"} + ], + "usage": _usage(), + } + ).encode() + ) + chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"} + deltas: Final[tuple[dict[str, JsonValue], ...]] = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": marker}}]}, + {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**chunk, "choices": [], "usage": _usage()}, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(b"data: " + json.dumps(delta).encode() + b"\n\n" for delta in deltas), b"data: [DONE]\n\n"), + ) + + +def _responses_reply(marker: str, stream: bool) -> Reply: + identity: Final = uuid.uuid4().hex + response: Final[dict[str, JsonValue]] = { + "id": f"resp_{identity}", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": f"msg_{identity}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": marker, "annotations": []}], + } + ], + "usage": {"input_tokens": 7, "output_tokens": 2, "total_tokens": 9}, + } + if not stream: + return Reply(body=json.dumps(response).encode()) + events: Final[tuple[dict[str, JsonValue], ...]] = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**response, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": f"msg_{identity}", + "output_index": 0, + "content_index": 0, + "delta": marker, + }, + {"type": "response.completed", "sequence_number": 2, "response": response}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events), + ) + + +def _upstream(request: Request) -> Reply: + found: Final = MARKER.search(request.body) + if found is None: + return Reply(status=404, body=b'{"error":"no marker"}') + marker: Final = found.group(0).decode() + if marker.startswith("cmc-fail-"): + return Reply(status=500, body=b'{"error":{"message":"scripted upstream failure","type":"server_error"}}') + stream: Final = object_value(JSON.validate_json(request.body)).get("stream") is True + if request.target.endswith("/responses"): + return _responses_reply(marker, stream) + return _chat_reply(marker, stream) + + +def _body(model: str, endpoint: Endpoint, marker: str, stream: bool) -> tuple[str, dict[str, JsonValue]]: + if endpoint == "responses": + return "/v1/responses", {"model": model, "input": marker, "stream": stream} + messages: Final[list[JsonValue]] = [{"role": "user", "content": marker}] + if endpoint == "messages": + return "/v1/messages", {"model": model, "max_tokens": 16, "messages": messages, "stream": stream} + return "/v1/chat/completions", {"model": model, "messages": messages, "stream": stream} + + +@dataclass(frozen=True, slots=True) +class Sent: + marker: str + trace_id: str + call_id: str + status: int + text: str + + +@dataclass(frozen=True, slots=True) +class Cursors: + operator: int + tenant: int + arize: int + newrelic: int + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + owned: OwnedProxy + scenario: Scenario + model: str + upstream: Wire + sinks: SpanSinks + newrelic: ConnectSink + + def cursors(self) -> Cursors: + self.upstream.drain() + return Cursors( + recorded_spans(self.sinks.operator)[0], + recorded_spans(self.sinks.tenant)[0], + recorded_spans(self.sinks.arize)[0], + recorded_spans(self.newrelic.control_url)[0], + ) + + def upstream_hits(self, marker: str) -> int: + return sum(1 for request in self.upstream.drain() if marker.encode() in request.body) + + def langfuse(self, name: str, host: str | None = None) -> dict[str, str]: + return { + "langfuse_public_key": f"pk-lf-{name}", + "langfuse_secret_key": f"sk-lf-{name}", + "langfuse_host": host or self.sinks.tenant, + } + + def attach(self, team: str, callback: str, variables: Mapping[str, str], kind: str | None = None) -> httpx.Response: + body: Final[dict[str, JsonValue]] = {"callback_name": callback, "callback_vars": dict(variables)} + return self.proxy.request( + "POST", f"/team/{team}/callback", body if kind is None else {**body, "callback_type": kind} + ) + + def team_key( + self, + capture: Capture | None, + name: str, + *, + callback: str = "langfuse_otel", + variables: Mapping[str, str] | None = None, + kind: str | None = None, + ) -> tuple[str, str]: + team: Final = self.scenario.team() + base: Final = dict(variables) if variables is not None else self.langfuse(name) + attached: Final = self.attach( + team, callback, base if capture is None else {**base, "capture_message_content": capture}, kind + ) + assert attached.status_code == 200, attached.text + return team, self.scenario.key(team_id=team) + + def send( + self, + key: str, + endpoint: Endpoint = "chat", + stream: bool = False, + *, + marker: str | None = None, + extra: Mapping[str, JsonValue] | None = None, + ) -> Sent: + tagged: Final = marker or _marker() + path, body = _body(self.model, endpoint, tagged, stream) + trace_id: Final = uuid.uuid4().hex + call_id: Final = str(uuid.uuid4()) + headers: Final = { + "Authorization": f"Bearer {key}", + "traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01", + "x-litellm-call-id": call_id, + } + with self.proxy.client.stream("POST", path, json={**body, **(extra or {})}, headers=headers) as response: + text: Final = response.read().decode() + return Sent(tagged, trace_id, call_id, response.status_code, text) + + +def _served(sent: Sent) -> Sent: + assert sent.status == 200, sent.text + assert sent.marker in sent.text, sent.text + return sent + + +Credential = tuple[str, str] + + +def _sent_with(url: str, credential: Credential) -> frozenset[str]: + """Span ids ``url`` received in export requests whose ``credential`` header matched.""" + header, value = credential + return frozenset( + span_id + for request in recorded_requests(url) + if {name.lower(): seen for name, seen in object_value(request.get("headers") or {}).items()}.get(header) + == value + for span_id in request.get("span_ids") or () + if isinstance(span_id, str) + ) + + +def _of_request(url: str, since: int, sent: Sent, credential: Credential | None = None) -> tuple[Span, ...]: + """Spans of ``sent``'s trace at ``url``, narrowed to one account's export requests when ``credential`` is set.""" + spans: Final = recorded_spans(url, since)[1] + traces: Final = {sent.trace_id} | { + span["trace_id"] for span in spans if span["attributes"].get(CALL_ID) == sent.call_id + } + owned: Final = None if credential is None else _sent_with(url, credential) + return tuple(span for span in spans if span["trace_id"] in traces and (owned is None or span["span_id"] in owned)) + + +def _carries(span: Span, marker: str) -> bool: + return marker in json.dumps(span["attributes"]) + + +def _model_spans(spans: tuple[Span, ...]) -> tuple[Span, ...]: + return tuple(span for span in spans if OPERATION in span["attributes"]) + + +def _names(spans: tuple[Span, ...]) -> list[str]: + return sorted(span["name"] for span in spans) + + +def _with_content(url: str, since: int, sent: Sent) -> tuple[Span, ...]: + """The copy of ``sent`` at ``url`` once its model span arrived and some span carries the marker.""" + return eventually( + lambda: _of_request(url, since, sent), + lambda spans: bool(_model_spans(spans)) and any(_carries(span, sent.marker) for span in spans), + seconds=40, + ) + + +def _arrived( + url: str, since: int, sent: Sent, expected: frozenset[str], credential: Credential | None = None +) -> tuple[Span, ...]: + """The copy of ``sent`` at ``url`` once every span name in ``expected`` and a usage-bearing model span landed.""" + spans: Final = eventually( + lambda: _of_request(url, since, sent, credential), + lambda found: ( + expected <= {span["name"] for span in found} + and any(INPUT_TOKENS in span["attributes"] for span in _model_spans(found)) + ), + seconds=40, + ) + assert all( + span["attributes"].get(INPUT_TOKENS) == 7 for span in _model_spans(spans) if INPUT_TOKENS in span["attributes"] + ) + return spans + + +def _assert_redacted_twin( + rig: Rig, sent: Sent, cursors: Cursors, sink: str, since: int, credential: Credential | None = None +) -> tuple[Span, ...]: + operator: Final = _with_content(rig.sinks.operator, cursors.operator, sent) + carried: Final = frozenset(span["name"] for span in operator if _carries(span, sent.marker)) + tenant: Final = _arrived(sink, since, sent, carried, credential) + leaked: Final = [span["name"] for span in tenant if _carries(span, sent.marker)] + assert leaked == [], f"{leaked} carried content; tenant saw {_names(tenant)}" + model: Final = _model_spans(tenant)[0]["attributes"] + assert "gen_ai.input.messages" not in model and "gen_ai.output.messages" not in model, sorted(model) + assert model.get("gen_ai.request.model"), sorted(model) + return tenant + + +def _assert_content_twin(rig: Rig, sent: Sent, cursors: Cursors) -> None: + operator: Final = _with_content(rig.sinks.operator, cursors.operator, sent) + carried: Final = frozenset(span["name"] for span in operator if _carries(span, sent.marker)) + tenant: Final = _arrived(rig.sinks.tenant, cursors.tenant, sent, carried) + kept: Final = frozenset(span["name"] for span in tenant if _carries(span, sent.marker)) + assert carried <= kept, f"content lost on {sorted(carried - kept)}" + + +@pytest.mark.parametrize( + ("endpoint", "stream"), + [("chat", False), ("chat", True), ("responses", False), ("messages", True)], + ids=["chat-unary", "chat-stream", "responses-unary", "messages-stream"], +) +def test_no_content_team_keeps_its_trace_and_metadata_without_content( + rig: Rig, endpoint: Endpoint, stream: bool +) -> None: + _, key = rig.team_key("no_content", "a") + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key, endpoint, stream)) + assert rig.upstream_hits(sent.marker) == 1 + _assert_redacted_twin(rig, sent, cursors, rig.sinks.tenant, cursors.tenant) + + +@pytest.mark.parametrize("capture", ["span_only", None], ids=["span_only", "omitted"]) +def test_span_only_and_omitted_teams_keep_the_globally_captured_content(rig: Rig, capture: Capture | None) -> None: + _, key = rig.team_key(capture, "b") + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + _assert_content_twin(rig, sent, cursors) + + +def test_tool_call_arguments_never_reach_a_no_content_team(rig: Rig) -> None: + _, key = rig.team_key("no_content", "a") + cursors: Final = rig.cursors() + tool: Final[dict[str, JsonValue]] = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}}, + } + sent: Final = rig.send(key, marker=_marker("tool-"), extra={"tools": [tool]}) + assert sent.status == 200 and sent.marker in sent.text, sent.text + operator: Final = _with_content(rig.sinks.operator, cursors.operator, sent) + assert any(sent.marker in str(span["attributes"].get("gen_ai.output.messages", "")) for span in operator), ( + "operator copy lacks the tool-call arguments" + ) + tenant: Final = _assert_redacted_twin(rig, sent, cursors, rig.sinks.tenant, cursors.tenant) + assert "lookup" in json.dumps([span["attributes"] for span in tenant]), "tool definitions are metadata and stay" + + +def test_cache_hit_twin_of_a_no_content_team_is_redacted(rig: Rig) -> None: + _, key = rig.team_key("no_content", "a") + marker: Final = _marker() + _served(rig.send(key, marker=marker)) + assert rig.upstream_hits(marker) == 1 + cursors: Final = rig.cursors() + hit: Final = eventually( + lambda: _served(rig.send(key, marker=marker)), lambda _: rig.upstream_hits(marker) == 0, seconds=20 + ) + operator: Final = eventually( + lambda: _of_request(rig.sinks.operator, cursors.operator, hit), + lambda spans: any(span["kind"] == SERVER for span in spans), + seconds=40, + ) + expected: Final = frozenset(span["name"] for span in operator if span["kind"] == SERVER) + tenant: Final = eventually( + lambda: _of_request(rig.sinks.tenant, cursors.tenant, hit), + lambda spans: expected <= {span["name"] for span in spans}, + seconds=40, + ) + assert [span["name"] for span in tenant if _carries(span, marker)] == [], _names(tenant) + + +def test_a_failure_only_entry_rejects_the_setting(rig: Rig) -> None: + team: Final = rig.scenario.team() + response: Final = rig.attach( + team, "langfuse_otel", {**rig.langfuse("fail"), "capture_message_content": "no_content"}, "failure" + ) + assert response.status_code == 400, response.text + assert "success_and_failure" in response.text, response.text + + +def test_key_level_logging_with_no_content_is_redacted(rig: Rig) -> None: + logging: Final[list[JsonValue]] = [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {**rig.langfuse("key"), "capture_message_content": "no_content"}, + } + ] + key: Final = rig.scenario.key(metadata={"logging": logging}) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + _assert_redacted_twin(rig, sent, cursors, rig.sinks.tenant, cursors.tenant) + + +def test_same_account_destination_redacts_the_shared_copy_only_for_that_team(rig: Rig) -> None: + _, restricted = rig.team_key("no_content", "operator", variables=rig.langfuse("operator", host=rig.sinks.operator)) + _, other = rig.team_key("span_only", "b") + cursors: Final = rig.cursors() + shared: Final = _served(rig.send(restricted)) + assert rig.upstream_hits(shared.marker) == 1 + copy: Final = eventually( + lambda: _of_request(rig.sinks.operator, cursors.operator, shared), + lambda spans: any(INPUT_TOKENS in span["attributes"] for span in _model_spans(spans)), + seconds=40, + ) + assert [span["name"] for span in copy if _carries(span, shared.marker)] == [], _names(copy) + assert len([span for span in copy if span["kind"] == SERVER and span["name"].startswith("POST ")]) == 1, _names( + copy + ) + _assert_content_twin(rig, _served(rig.send(other)), cursors) + + +def test_arize_destination_drops_openinference_content(rig: Rig) -> None: + _, key = rig.team_key( + "no_content", "arize", callback="arize", variables={"arize_space_key": "space-a", "arize_api_key": "arize-a"} + ) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + operator: Final = _with_content(rig.sinks.operator, cursors.operator, sent) + assert any(sent.marker in str(span["attributes"].get("input.value", "")) for span in operator), ( + "operator copy has no OpenInference content to strip" + ) + tenant: Final = _assert_redacted_twin( + rig, sent, cursors, rig.sinks.arize, cursors.arize, ("arize-space-id", "space-a") + ) + model: Final = _model_spans(tenant)[0]["attributes"] + assert not any(name.startswith(("llm.input_messages.", "llm.output_messages.")) for name in model), sorted(model) + assert "input.value" not in model and "output.value" not in model, sorted(model) + assert model.get("llm.model_name"), sorted(model) + + +def test_weave_destination_drops_its_content(rig: Rig) -> None: + _, key = rig.team_key( + "no_content", + "weave", + callback="weave_otel", + variables={"wandb_api_key": "wandb-a", "weave_project_id": "team/a"}, + ) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + operator: Final = eventually( + lambda: _of_request(rig.sinks.arize, cursors.arize, sent, ("project_id", "operator/weave")), + lambda spans: any(sent.marker in str(span["attributes"].get("weave.output", "")) for span in spans), + seconds=40, + ) + assert operator, "operator weave copy has no weave.output to strip" + tenant: Final = _assert_redacted_twin(rig, sent, cursors, rig.sinks.arize, cursors.arize, ("project_id", "team/a")) + assert not any("weave.output" in span["attributes"] for span in tenant), _names(tenant) + + +def test_newrelic_destination_drops_genai_content(rig: Rig) -> None: + _, key = rig.team_key("no_content", "newrelic", callback="newrelic", variables={"newrelic_api_key": "nr-team-a"}) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + tenant: Final = _assert_redacted_twin( + rig, sent, cursors, rig.newrelic.control_url, cursors.newrelic, ("api-key", "nr-team-a") + ) + hosts: Final = { + object_value(request.get("headers") or {}).get("Host") + for request in recorded_requests(rig.newrelic.control_url) + if object_value(request.get("headers") or {}).get("api-key") == "nr-team-a" + } + assert hosts == {"otlp.nr-data.net"}, hosts + assert tenant + + +def test_span_only_team_lifts_global_no_content_only_for_its_destination(dark_rig: Rig) -> None: + """The sibling Arize entry gets a fan-out destination and the SigNoz entry rides the routed tracer.""" + team, key = dark_rig.team_key("span_only", "b") + siblings: Final = ( + dark_rig.attach(team, "arize", {"arize_space_key": "space-sibling", "arize_api_key": "arize-sibling"}), + dark_rig.attach( + team, + "signoz", + {"signoz_ingestion_endpoint": dark_rig.sinks.arize + "/v1/traces", "signoz_ingestion_key": "signoz-team"}, + ), + ) + assert [response.status_code for response in siblings] == [200, 200], [response.text for response in siblings] + cursors: Final = dark_rig.cursors() + sent: Final = _served(dark_rig.send(key)) + assert dark_rig.upstream_hits(sent.marker) == 1 + operator: Final = _arrived(dark_rig.sinks.operator, cursors.operator, sent, frozenset()) + tenant: Final = _with_content(dark_rig.sinks.tenant, cursors.tenant, sent) + sibling: Final = _arrived( + dark_rig.sinks.arize, + cursors.arize, + sent, + frozenset(), + ("arize-space-id", "space-sibling"), + ) + signoz: Final = _arrived( + dark_rig.sinks.arize, + cursors.arize, + sent, + frozenset(), + ("signoz-ingestion-key", "signoz-team"), + ) + assert any(_carries(span, sent.marker) for span in tenant), _names(tenant) + assert not any(_carries(span, sent.marker) for span in operator), _names(operator) + assert not any(_carries(span, sent.marker) for span in sibling), _names(sibling) + assert not any(_carries(span, sent.marker) for span in signoz), _names(signoz) + + +def test_an_unsupported_value_fails_registration(rig: Rig) -> None: + team: Final = rig.scenario.team() + response: Final = rig.attach(team, "langfuse_otel", {**rig.langfuse("a"), "capture_message_content": "bogus"}) + assert response.status_code == 422, response.text + assert "Invalid capture_message_content 'bogus'" in response.text, response.text + + +def test_classic_langfuse_rejects_the_setting(rig: Rig) -> None: + team: Final = rig.scenario.team() + response: Final = rig.attach(team, "langfuse", {**rig.langfuse("a"), "capture_message_content": "no_content"}) + assert response.status_code == 400, response.text + assert "capture_message_content" in response.text, response.text + + +def test_a_second_entry_for_the_same_backend_cannot_disagree(rig: Rig) -> None: + team, _ = rig.team_key("no_content", "a") + response: Final = rig.attach(team, "langfuse_otel", {"capture_message_content": "span_only"}, "success_and_failure") + assert response.status_code == 400, response.text + assert "already set to 'no_content' by another langfuse_otel entry" in response.text, response.text + + +def test_replacing_the_registration_switches_the_team_to_content(rig: Rig) -> None: + team, key = rig.team_key("no_content", "a") + cursors: Final = rig.cursors() + _assert_redacted_twin(rig, _served(rig.send(key)), cursors, rig.sinks.tenant, cursors.tenant) + removed: Final = rig.proxy.request("DELETE", f"/team/{team}/callback/langfuse_otel") + assert removed.status_code == 200, removed.text + replaced: Final = rig.attach(team, "langfuse_otel", {**rig.langfuse("a"), "capture_message_content": "span_only"}) + assert replaced.status_code == 200, replaced.text + + def attempt() -> bool: + since: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + operator: Final = _with_content(rig.sinks.operator, since.operator, sent) + carried: Final = frozenset(span["name"] for span in operator if _carries(span, sent.marker)) + tenant: Final = _arrived(rig.sinks.tenant, since.tenant, sent, carried) + return any(_carries(span, sent.marker) for span in tenant) + + assert eventually(attempt, bool, seconds=60) + + +def test_request_metadata_cannot_lift_the_team_restriction(rig: Rig) -> None: + _, key = rig.team_key("no_content", "a") + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key, extra={"metadata": {"capture_message_content": "span_only"}})) + assert rig.upstream_hits(sent.marker) == 1 + _assert_redacted_twin(rig, sent, cursors, rig.sinks.tenant, cursors.tenant) + + +def test_stalled_tenant_during_a_mixed_burst_keeps_serving_and_each_teams_policy(rig: Rig) -> None: + _, restricted = rig.team_key("no_content", "a") + _, open_key = rig.team_key("span_only", "b") + shapes: Final[tuple[tuple[Endpoint, bool], ...]] = ( + ("chat", False), + ("chat", True), + ("responses", False), + ("messages", True), + ) + plan: Final = [(restricted if index % 2 else open_key, *shapes[index % len(shapes)]) for index in range(30)] + cursors: Final = rig.cursors() + configure_sink(rig.sinks.tenant, paused=True) + try: + with ThreadPoolExecutor(max_workers=10) as pool: + sent: Final = list(pool.map(lambda job: rig.send(job[0], job[1], job[2]), plan)) + finally: + configure_sink(rig.sinks.tenant, paused=False) + assert [result.status for result in sent] == [200] * 30, [result.text for result in sent if result.status != 200] + for (key, _, _), result in zip(plan, sent, strict=True): + roots = [ + span + for span in _with_content(rig.sinks.operator, cursors.operator, result) + if span["kind"] == SERVER and span["name"].startswith("POST ") + ] + assert len(roots) == 1, roots + if key == restricted: + _assert_redacted_twin(rig, result, cursors, rig.sinks.tenant, cursors.tenant) + else: + _assert_content_twin(rig, result, cursors) + + +def _write_config(directory: Path, sinks: SpanSinks, name: str, callbacks: tuple[str, ...]) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["litellm_settings"] = { + **config.get("litellm_settings", {}), + "callbacks": list(callbacks), + "otel_tenant_destination_mode": "additive", + "provider_url_destination_allowed_hosts": [ + urlparse(sinks.tenant).netloc, + urlparse(sinks.operator).netloc, + urlparse(sinks.arize).netloc, + ], + } + config["general_settings"] = {**config.get("general_settings", {}), "user_api_key_cache_ttl": 2} + path: Final = directory / f"{name}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@contextmanager +def _started( + provider: Wire, + sinks: SpanSinks, + newrelic: ConnectSink, + directory: Path, + capture: str | None, + callbacks: tuple[str, ...] = ("langfuse_otel",), +) -> Generator[Rig]: + environment: Final = { + "LITELLM_OTEL_V2": "1", + "OTEL_BSP_SCHEDULE_DELAY": "300", + "MAPPER_NAMES": "genai,openinference", + "LANGFUSE_HOST": sinks.operator, + "LANGFUSE_PUBLIC_KEY": "pk-lf-operator", + "LANGFUSE_SECRET_KEY": "sk-lf-operator", + "ARIZE_SPACE_KEY": "space-operator", + "ARIZE_API_KEY": "arize-operator", + "ARIZE_HTTP_ENDPOINT": sinks.arize + "/v1/traces", + "WANDB_API_KEY": "wandb-operator", + "WANDB_PROJECT_ID": "operator/weave", + "WANDB_HOST": sinks.arize, + "SIGNOZ_INGESTION_ENDPOINT": sinks.arize + "/v1/traces", + "SIGNOZ_INGESTION_KEY": "signoz-operator", + "HTTPS_PROXY": newrelic.proxy_url, + "NO_PROXY": "127.0.0.1,localhost", + "REQUESTS_CA_BUNDLE": newrelic.ca_pem, + **({} if capture is None else {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": capture}), + } + with ( + gateway_from_environment() as gateway, + owned_proxy_process( + gateway, + directory, + environment, + config=_write_config(directory, sinks, "capture", callbacks), + remove_environment=( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", + "LITELLM_OTEL_TENANT_DESTINATION_MODE", + ), + workers=2, + ) as owned, + owned.gateway.scenario() as scenario, + ): + yield Rig( + owned.gateway, owned, scenario, scenario.model(api_base=provider.url + "/v1"), provider, sinks, newrelic + ) + + +@pytest.fixture(scope="module") +def provider() -> Iterator[Wire]: + with wire_server(_upstream) as wire: + yield wire + + +@pytest.fixture(scope="module") +def newrelic_sink(tmp_path_factory: pytest.TempPathFactory) -> Iterator[ConnectSink]: + with owned_connect_sink(tmp_path_factory.mktemp("newrelic-sink")) as sink: + yield sink + + +@pytest.fixture(scope="module") +def rig( + provider: Wire, audit_sinks: SpanSinks, newrelic_sink: ConnectSink, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("capture-span-only") + with _started(provider, audit_sinks, newrelic_sink, directory, "span_only") as started: + yield started + + +@pytest.fixture(scope="module") +def dark_rig( + provider: Wire, audit_sinks: SpanSinks, newrelic_sink: ConnectSink, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Rig]: + with _started( + provider, + audit_sinks, + newrelic_sink, + tmp_path_factory.mktemp("capture-no-content"), + None, + ("langfuse_otel", "signoz"), + ) as started: + yield started diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py index f0e7163e264..1104a6151db 100644 --- a/tests/integration/providers/test_decisions_openai_format_wire.py +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -105,19 +105,25 @@ class _Provider: wraps_result: bool cost_map_key: str | None speaks_openai: bool = False + provider_reported_cost: float | None = None def upstream_body(self) -> dict[str, JsonValue]: if self.speaks_openai: return {"model": self.body_model, "input": _INPUT, "questions": _QUESTIONS} return {"model": self.body_model, "state": _INPUT, "questions": _SYSTEM_ONE_QUESTIONS} + def _with_reported_cost(self, usage: dict[str, JsonValue]) -> dict[str, JsonValue]: + if self.provider_reported_cost is None: + return usage + return {**usage, "cost": self.provider_reported_cost} + def upstream_reply(self) -> dict[str, JsonValue]: if self.speaks_openai: - return {"model": self.body_model, "answers": _ANSWERS, "usage": _USAGE} + return {"model": self.body_model, "answers": _ANSWERS, "usage": self._with_reported_cost(_USAGE)} answer: Final[dict[str, JsonValue]] = { "model": self.body_model, "answers": _SYSTEM_ONE_ANSWERS, - "usage": _SYSTEM_ONE_USAGE, + "usage": self._with_reported_cost(_SYSTEM_ONE_USAGE), } return {"result": answer, "success": True} if self.wraps_result else answer @@ -143,7 +149,8 @@ _PROVIDERS: Final = ( "typesafe/jev-1.13", _API_KEY, False, - "openrouter/typesafe/jev-1.13", + None, + provider_reported_cost=1.5834e-5, ), _Provider( "strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None @@ -235,11 +242,13 @@ def _number(value: JsonValue) -> float: return float(value) -def _expected_spend(cost_map_key: str | None) -> float: - if cost_map_key is None: +def _expected_spend(provider: _Provider) -> float: + if provider.provider_reported_cost is not None: + return provider.provider_reported_cost + if provider.cost_map_key is None: return 0.0 cost_map: Final = _JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes()) - prices: Final = object_value(cost_map[cost_map_key]) + prices: Final = object_value(cost_map[provider.cost_map_key]) return _INPUT_TOKENS * _number(prices["input_cost_per_token"]) + _OUTPUT_TOKENS * _number( prices["output_cost_per_token"] ) @@ -320,10 +329,8 @@ def _drop_params_config(directory: Path, api_base: str) -> Path: @pytest.mark.parametrize("provider", _PROVIDERS, ids=_provider_id) -def test_each_provider_gets_its_own_path_key_and_body_and_is_billed_from_the_cost_map( - gateway: Gateway, provider: _Provider -) -> None: - expected_spend: Final = _expected_spend(provider.cost_map_key) +def test_each_provider_gets_its_own_path_key_and_body_and_is_billed(gateway: Gateway, provider: _Provider) -> None: + expected_spend: Final = _expected_spend(provider) with gateway.scenario() as scenario: handle: Final = _register(scenario, provider.upstream_reply()) model: Final = _deployment(scenario, handle, provider) diff --git a/tests/unit/experimental_mcp_client/test_mcp_client.py b/tests/unit/experimental_mcp_client/test_mcp_client.py index ff192c9279c..f949d2d3e58 100644 --- a/tests/unit/experimental_mcp_client/test_mcp_client.py +++ b/tests/unit/experimental_mcp_client/test_mcp_client.py @@ -6,7 +6,7 @@ import os import selectors import sys from collections.abc import AsyncIterator, Callable -from contextlib import asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager from pathlib import Path from types import ModuleType from typing import Final @@ -15,15 +15,21 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch import anyio import httpx2 import pytest -from mcp import MCPError +from mcp import ClientSession, MCPError +from mcp.client.session import ClientRequestContext, ElicitationFnT from mcp.client.streamable_http import streamable_http_client +from mcp.server.mcpserver import Context +from mcp.server.mcpserver import MCPServer as UpstreamServer +from mcp.server.transport_security import TransportSecuritySettings from mcp.shared.message import SessionMessage from mcp.types import ( CONNECTION_CLOSED, - INTERNAL_ERROR, REQUEST_TIMEOUT, CallToolRequestParams, CallToolResult, + DiscoverResult, + ElicitRequestParams, + ElicitResult, ErrorData, Implementation, InitializeResult, @@ -35,12 +41,20 @@ from mcp.types import ( ServerCapabilities, ) from mcp_types.version import LATEST_HANDSHAKE_VERSION -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from starlette.applications import Starlette +from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint +from starlette.requests import Request +from starlette.responses import Response +from starlette.types import ASGIApp, Message, Receive, Scope, Send # Add the parent directory to the path so we can import litellm import litellm.experimental_mcp_client.client as mcp_client_module from litellm.experimental_mcp_client.client import ( MCPClient, + MCPSigV4Auth, + PersistentMCPSession, + UpstreamSessionClosedError, _first_non_cancelled_cause, _TransportContext, _TransportStreams, @@ -137,13 +151,15 @@ async def test_modern_input_request_uses_existing_elicitation_callback( modern_caller: bool, sampling: bool, elicitation_mode: str ) -> None: from queue import SimpleQueue + from mcp.types import ( - ElicitResult, - ElicitRequestParams, - TextContent, CreateMessageRequestParams, CreateMessageResult, + ElicitRequestParams, + ElicitResult, + TextContent, ) + from litellm.proxy._experimental.mcp_server.interactions import BoundInputRequiredResult observed: Final[SimpleQueue[str]] = SimpleQueue() @@ -841,7 +857,9 @@ class TestExecuteSessionOperationSurfacesTransportError: raise _FakeExceptionGroup("transport", [_FakeExceptionGroup("reader", failures)]) self._make_session(session_class, initialize) - expected: Final = close_error if failure_phase == "early" else connect_error if failure_phase == "mixed" else cancelled + expected: Final = ( + close_error if failure_phase == "early" else connect_error if failure_phase == "mixed" else cancelled + ) with pytest.raises(type(expected)) as caught: await client._execute_session_operation(self._make_transport(close_transport), AsyncMock(), http_client) assert caught.value is expected @@ -861,7 +879,6 @@ class TestExecuteSessionOperationSurfacesTransportError: result = await client._execute_session_operation(transport_ctx, _op) assert result == "done" - @pytest.mark.asyncio @patch("litellm.experimental_mcp_client.client.ClientSession") async def test_session_entry_failure_still_closes_transport(self, session_class): @@ -1883,7 +1900,9 @@ async def test_http_response_handler_preserves_success_and_http_errors(status_co @pytest.mark.asyncio async def test_http_status_check_allows_auth_refresh_before_rejecting() -> None: - from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ClientCredentialsBearerAuth + from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ( + ClientCredentialsBearerAuth, + ) seen = [] @@ -2146,14 +2165,21 @@ def test_sse_read_failure_is_preserved() -> None: @pytest.mark.parametrize("protocol_version", ["auto", "2025-06-18"]) @pytest.mark.parametrize("transport", [MCPTransport.sse, MCPTransport.stdio]) @pytest.mark.parametrize("mode", ["ok", "closed", "silent"]) -async def test_transport_completion_and_normal_messages(transport: MCPTransport, mode: str, protocol_version: str) -> None: +async def test_transport_completion_and_normal_messages( + transport: MCPTransport, mode: str, protocol_version: str +) -> None: from mcp import ClientSession + from litellm.proxy._experimental.mcp_server.rest_endpoints import _connection_error_message logging_callback: Final = AsyncMock() read_timeout: Final = 0.2 if mode == "silent" else 30 client: Final = MCPClient( - server_url="https://example.com/sse", transport_type=transport, timeout=read_timeout, logging_callback=logging_callback, protocol_version=protocol_version + server_url="https://example.com/sse", + transport_type=transport, + timeout=read_timeout, + logging_callback=logging_callback, + protocol_version=protocol_version, ) async def operation(session: ClientSession) -> CallToolResult: @@ -2735,8 +2761,15 @@ def test_client_import_before_proxy_credentials_succeeds_in_fresh_process(): import subprocess result = subprocess.run( - [sys.executable, "-c", "import litellm.experimental_mcp_client.client; from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager; print(MCPServerManager.__name__)"], - capture_output=True, text=True, timeout=60, check=False, + [ + sys.executable, + "-c", + "import litellm.experimental_mcp_client.client; from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager; print(MCPServerManager.__name__)", + ], + capture_output=True, + text=True, + timeout=60, + check=False, ) assert result.returncode == 0, result.stderr assert result.stdout.strip() == "MCPServerManager" @@ -2764,13 +2797,36 @@ async def test_discovery_auth_fingerprint_tracks_effective_credentials(resolved: assert "private-original-credential" not in original +@pytest.mark.asyncio +async def test_discovery_auth_fingerprint_tracks_aws_sigv4_credential_identity() -> None: + def client(access_key_id: str) -> MCPClient: + return MCPClient( + server_url="https://example.com/mcp", + auth_type=MCPAuth.aws_sigv4, + aws_auth=MCPSigV4Auth( + aws_access_key_id=access_key_id, + aws_secret_access_key="private-secret", + aws_region_name="us-east-1", + ), + ) + + original: Final = await client("AKIAORIGINAL").discovery_auth_fingerprint() + repeated: Final = await client("AKIAORIGINAL").discovery_auth_fingerprint() + rotated: Final = await client("AKIAROTATED").discovery_auth_fingerprint() + assert original == repeated + assert original != rotated + assert "AKIAORIGINAL" not in original + + @pytest.mark.asyncio async def test_request_auth_preview_uses_the_same_effective_headers_as_egress() -> None: from litellm.proxy._experimental.mcp_server.outbound_credentials.httpx_auth import StaticHeaderAuth client: Final = MCPClient( - server_url="https://upstream.example/mcp", auth_type=MCPAuth.bearer_token, - resolved_auth=StaticHeaderAuth("Bearer resolved"), extra_headers={"X-Trace": "trace"}, + server_url="https://upstream.example/mcp", + auth_type=MCPAuth.bearer_token, + resolved_auth=StaticHeaderAuth("Bearer resolved"), + extra_headers={"X-Trace": "trace"}, ) request: Final = await client.prepare_request_auth() assert request.method == "POST" @@ -2794,18 +2850,29 @@ async def test_expired_session_preserves_sdk_error_and_next_operation_reinitiali return httpx2.Response(202) requests.append((payload["method"], request.headers.get("mcp-session-id"))) if payload["method"] == "initialize": - return httpx2.Response(200, headers={"mcp-session-id": f"session-{len(requests)}"}, json={ - "jsonrpc": "2.0", "id": payload["id"], "result": { - "protocolVersion": "2025-06-18", "capabilities": {}, - "serverInfo": {"name": "expiry-test", "version": "1"}, + return httpx2.Response( + 200, + headers={"mcp-session-id": f"session-{len(requests)}"}, + json={ + "jsonrpc": "2.0", + "id": payload["id"], + "result": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "serverInfo": {"name": "expiry-test", "version": "1"}, + }, }, - }) + ) if len(requests) == 2: if rpc_error: - return httpx2.Response(404, json={ - "jsonrpc": "2.0", "id": payload["id"], - "error": {"code": METHOD_NOT_FOUND, "message": "Tool catalog unavailable"}, - }) + return httpx2.Response( + 404, + json={ + "jsonrpc": "2.0", + "id": payload["id"], + "error": {"code": METHOD_NOT_FOUND, "message": "Tool catalog unavailable"}, + }, + ) return httpx2.Response(404) return httpx2.Response(200, json={"jsonrpc": "2.0", "id": payload["id"], "result": {"tools": []}}) @@ -2821,7 +2888,12 @@ async def test_expired_session_preserves_sdk_error_and_next_operation_reinitiali streamable_http_client(client.server_url, http_client=http_client), lambda session: session.list_tools() ) assert result.tools == [] - assert requests == [("initialize", None), ("tools/list", "session-1"), ("initialize", None), ("tools/list", "session-3")] + assert requests == [ + ("initialize", None), + ("tools/list", "session-1"), + ("initialize", None), + ("tools/list", "session-3"), + ] @pytest.mark.asyncio @@ -3217,16 +3289,32 @@ async def test_configured_upstream_revision_is_offered_and_checked(revision, acc assert payload.params["protocolVersion"] == offered assert ("sampling" in payload.params["capabilities"]) == callbacks assert ("elicitation" in payload.params["capabilities"]) == callbacks - return httpx2.Response(200, json={ - "jsonrpc": "2.0", "id": payload.id, - "result": {"protocolVersion": offered if accepted else "unsupported", - "capabilities": {"tools": {}}, "serverInfo": {"name": "upstream", "version": "1"}}, - }) + return httpx2.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": { + "protocolVersion": offered if accepted else "unsupported", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "upstream", "version": "1"}, + }, + }, + ) assert accepted, "No operation may execute after failed version negotiation" - return httpx2.Response(200, json={"jsonrpc": "2.0", "id": payload.id, "result": {"tools": [{"name": "echo", "inputSchema": {"type": "object"}}]}}) + return httpx2.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": {"tools": [{"name": "echo", "inputSchema": {"type": "object"}}]}, + }, + ) client = _MockTransportClient( - respond, server_url="https://example.com/mcp", protocol_version=revision, + respond, + server_url="https://example.com/mcp", + protocol_version=revision, sampling_callback=AsyncMock() if callbacks else None, elicitation_callback=AsyncMock() if callbacks else None, ) @@ -3251,7 +3339,7 @@ def test_upstream_protocol_configuration_rejects_unavailable_modes(revision): async def test_modern_upstream_requests_are_self_contained_without_initialization(accepted: bool) -> None: from queue import SimpleQueue - from mcp.types import DiscoverResult, ToolsCapability + from mcp.types import ToolsCapability methods: Final[SimpleQueue[str]] = SimpleQueue() @@ -3340,7 +3428,7 @@ async def test_modern_call_emits_listed_argument_headers(paginated: bool, valid_ from collections.abc import Mapping from queue import SimpleQueue - from mcp.types import DiscoverResult, ToolsCapability + from mcp.types import ToolsCapability calls: Final[SimpleQueue[str]] = SimpleQueue() @@ -3424,6 +3512,7 @@ async def test_modern_call_emits_listed_argument_headers(paginated: bool, valid_ await asyncio.gather(call_as("Engineering"), call_as("Finance")) assert calls.qsize() == (2 if valid_annotation else 0) + @pytest.mark.asyncio @pytest.mark.parametrize("stop", ("page_cap", "repeated_cursor")) @pytest.mark.parametrize("schema_available", (False, True)) @@ -3432,7 +3521,7 @@ async def test_modern_call_requires_schema_from_bounded_listing( ) -> None: from queue import SimpleQueue - from mcp.types import DiscoverResult, ToolsCapability + from mcp.types import ToolsCapability monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_TOOL_LISTING_MAX_PAGES", 2) methods: Final[SimpleQueue[str]] = SimpleQueue() @@ -3508,7 +3597,7 @@ async def test_modern_call_requires_schema_from_bounded_listing( def test_modern_call_uses_discovery_deadline_for_multiple_pages() -> None: - from mcp.types import DiscoverResult, ToolsCapability + from mcp.types import ToolsCapability loop: Final = _AutojumpClockLoop() @@ -3585,7 +3674,7 @@ def test_modern_call_uses_discovery_deadline_for_multiple_pages() -> None: async def test_cancelled_modern_catalog_load_prevents_tool_execution() -> None: from queue import SimpleQueue - from mcp.types import DiscoverResult, ToolsCapability + from mcp.types import ToolsCapability listing_started: Final = asyncio.Event() hold_listing: Final = asyncio.Event() @@ -3624,20 +3713,30 @@ async def test_cancelled_modern_catalog_load_prevents_tool_execution() -> None: observed_listing.cancel() await asyncio.gather(task, observed_listing, return_exceptions=True) + def test_modern_upstream_rejects_legacy_sse_transport() -> None: with pytest.raises(ValueError, match="transport"): MCPClient(protocol_version="2026-07-28", transport_type=MCPTransport.sse) @pytest.mark.asyncio -@pytest.mark.parametrize("method, field, item", [ - ("tools/list", "tools", {"name": "second", "inputSchema": {"type": "object"}}), - ("prompts/list", "prompts", {"name": "second"}), - ("resources/list", "resources", {"name": "second", "uri": "status://second"}), - ("resources/templates/list", "resourceTemplates", {"name": "second", "uriTemplate": "status://{name}"}), -]) +@pytest.mark.parametrize( + "method, field, item", + [ + ("tools/list", "tools", {"name": "second", "inputSchema": {"type": "object"}}), + ("prompts/list", "prompts", {"name": "second"}), + ("resources/list", "resources", {"name": "second", "uri": "status://second"}), + ("resources/templates/list", "resourceTemplates", {"name": "second", "uriTemplate": "status://{name}"}), + ], +) async def test_single_catalog_page_preserves_cursor_metadata_and_request_cursor(method, field, item): - from mcp.types import ListToolsRequest, ListPromptsRequest, ListResourcesRequest, ListResourceTemplatesRequest, PaginatedRequestParams + from mcp.types import ( + ListPromptsRequest, + ListResourcesRequest, + ListResourceTemplatesRequest, + ListToolsRequest, + PaginatedRequestParams, + ) def respond(request: httpx2.Request) -> httpx2.Response: if request.method != "POST": @@ -3676,8 +3775,10 @@ async def test_single_catalog_page_preserves_cursor_metadata_and_request_cursor( client = _MockTransportClient(respond, server_url="https://upstream.example.com/mcp") request_type = { - "tools/list": ListToolsRequest, "prompts/list": ListPromptsRequest, - "resources/list": ListResourcesRequest, "resources/templates/list": ListResourceTemplatesRequest, + "tools/list": ListToolsRequest, + "prompts/list": ListPromptsRequest, + "resources/list": ListResourcesRequest, + "resources/templates/list": ListResourceTemplatesRequest, }[method] result = await client.list_page(request_type(params=PaginatedRequestParams(cursor="upstream-position"))) assert result.model_dump(by_alias=True)[field][0]["name"] == "second" @@ -3691,7 +3792,10 @@ async def test_single_catalog_page_preserves_cursor_metadata_and_request_cursor( @pytest.mark.parametrize("cursor", [None, "continuation"]) async def test_optional_catalog_distinguishes_absent_capability_from_failed_continuation(method, failure, cursor): from mcp.types import ( - ListPromptsRequest, ListResourcesRequest, ListResourceTemplatesRequest, PaginatedRequestParams, + ListPromptsRequest, + ListResourcesRequest, + ListResourceTemplatesRequest, + PaginatedRequestParams, ) methods = [] @@ -3704,19 +3808,35 @@ async def test_optional_catalog_distinguishes_absent_capability_from_failed_cont return httpx2.Response(202) methods.append(payload.method) if payload.method == "initialize": - return httpx2.Response(200, json={"jsonrpc": "2.0", "id": payload.id, "result": { - "protocolVersion": LATEST_HANDSHAKE_VERSION, - "capabilities": {} if failure == "unadvertised" else {"prompts": {}, "resources": {}}, - "serverInfo": {"name": "optional", "version": "1"}, - }}) + return httpx2.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": { + "protocolVersion": LATEST_HANDSHAKE_VERSION, + "capabilities": {} if failure == "unadvertised" else {"prompts": {}, "resources": {}}, + "serverInfo": {"name": "optional", "version": "1"}, + }, + }, + ) assert payload.method == method - return httpx2.Response(200, json={"jsonrpc": "2.0", "id": payload.id, "error": { - "code": -32601 if failure == "method_missing" else -32603, "message": "Upstream unavailable", - }}) + return httpx2.Response( + 200, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "error": { + "code": -32601 if failure == "method_missing" else -32603, + "message": "Upstream unavailable", + }, + }, + ) client = _MockTransportClient(respond, server_url="https://upstream.example.com/mcp") request_type = { - "prompts/list": ListPromptsRequest, "resources/list": ListResourcesRequest, + "prompts/list": ListPromptsRequest, + "resources/list": ListResourcesRequest, "resources/templates/list": ListResourceTemplatesRequest, }[method] request = request_type(params=PaginatedRequestParams(cursor=cursor)) @@ -3725,7 +3845,11 @@ async def test_optional_catalog_distinguishes_absent_capability_from_failed_cont await client.list_page(request) else: result = await client.list_page(request) - collection = {"prompts/list": "prompts", "resources/list": "resources", "resources/templates/list": "resource_templates"}[method] + collection: Final = { + "prompts/list": "prompts", + "resources/list": "resources", + "resources/templates/list": "resource_templates", + }[method] assert getattr(result, collection) == [] assert result.next_cursor is None if failure == "unadvertised": @@ -3884,3 +4008,937 @@ async def test_modern_elicitation_honors_server_permission(mode: str, enabled: b with pytest.raises(MCPError, match="Elicitation is disabled"): await client._request_with_interaction(session, request, None, None, True) request.assert_awaited_once_with(None, None) + + +class _StatefulUpstreamClient(MCPClient): + """An MCPClient whose streamable-HTTP transport talks to an in-process stateful MCP server.""" + + def __init__(self, app: ASGIApp, *, server_url: str, transport_type: MCPTransport) -> None: + super().__init__(server_url=server_url, transport_type=transport_type) + self._app = app + + def _create_transport_context(self) -> tuple[_TransportContext, httpx2.AsyncClient]: + http_client: Final = self._create_httpx_client_factory(transport=httpx2.ASGITransport(app=self._app))( + headers=self._get_auth_headers(), timeout=httpx2.Timeout(self.timeout) + ) + return streamable_http_client(self.server_url, http_client=http_client), http_client + + +class _CleanupHoldingASGITransport(httpx2.ASGITransport): + def __init__(self, app: ASGIApp, cleanup_started: asyncio.Event, release_cleanup: asyncio.Event) -> None: + super().__init__(app=app) + self._cleanup_started: Final = cleanup_started + self._release_cleanup: Final = release_cleanup + + async def aclose(self) -> None: + self._cleanup_started.set() + await self._release_cleanup.wait() + await super().aclose() + + +class _CleanupHoldingStatefulUpstreamClient(_StatefulUpstreamClient): + def __init__( + self, + app: ASGIApp, + *, + server_url: str, + transport_type: MCPTransport, + cleanup_started: asyncio.Event, + release_cleanup: asyncio.Event, + ) -> None: + super().__init__(app, server_url=server_url, transport_type=transport_type) + self._cleanup_started: Final = cleanup_started + self._release_cleanup: Final = release_cleanup + + def _create_transport_context(self) -> tuple[_TransportContext, httpx2.AsyncClient]: + transport: Final = _CleanupHoldingASGITransport( + self._app, + self._cleanup_started, + self._release_cleanup, + ) + http_client: Final = self._create_httpx_client_factory(transport=transport)( + headers=self._get_auth_headers(), + timeout=httpx2.Timeout(self.timeout), + ) + return streamable_http_client(self.server_url, http_client=http_client), http_client + + +class _InProcessUpstreamClient(MCPClient): + def __init__(self, server: UpstreamServer, *, elicitation_callback: ElicitationFnT | None = None) -> None: + super().__init__( + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + elicitation_callback=elicitation_callback, + ) + self._server = server + + def _create_transport_context(self) -> tuple[_TransportContext, httpx2.AsyncClient | None]: + @asynccontextmanager + async def connected() -> AsyncIterator[_TransportStreams]: + server_to_client_send, client_receive = anyio.create_memory_object_stream[SessionMessage | Exception](0) + client_to_server_send, server_receive = anyio.create_memory_object_stream[SessionMessage](0) + server_task: Final = asyncio.create_task( + self._server._lowlevel_server.run( + server_receive, + server_to_client_send, + self._server._lowlevel_server.create_initialization_options(), + ) + ) + try: + yield client_receive, client_to_server_send + finally: + await client_to_server_send.aclose() + await client_receive.aclose() + await asyncio.wait_for(server_task, 5) + + return connected(), None + + +class _ElicitationAnswer(BaseModel): + label: str + + model_config = ConfigDict(frozen=True) + + +def _stateful_upstream() -> Starlette: + service: Final = UpstreamServer("stateful") + selected: Final[dict[str, str]] = {} + + @service.tool() + def select_project(name: str, ctx: Context) -> str: + selected[ctx.headers["mcp-session-id"]] = name + return f"selected {name}" + + @service.tool() + def create_feature(title: str, ctx: Context) -> str: + return f"{selected[ctx.headers['mcp-session-id']]}/{title}" + + return service.streamable_http_app( + stateless_http=False, + json_response=True, + transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False), + ) + + +def _upstream_with_status_for_method(method: str, status_code: int) -> Starlette: + service: Final = UpstreamServer("status upstream") + + @service.tool() + def protected_tool() -> str: + return "protected" + + app: Final = service.streamable_http_app( + stateless_http=False, + json_response=True, + transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False), + ) + + async def reject_method(request: Request, call_next: RequestResponseEndpoint) -> Response: + if request.method == "POST": + payload: Final = await request.json() + if isinstance(payload, dict) and payload.get("method") == method: + return Response(status_code=status_code, headers={"WWW-Authenticate": "Bearer"}) + return await call_next(request) + + app.add_middleware(BaseHTTPMiddleware, dispatch=reject_method) + return app + + +def _upstream_with_malformed_response_for_method(method: str) -> Starlette: + service: Final = UpstreamServer("malformed response upstream") + + @service.tool() + def malformed_tool() -> str: + return "unreachable" + + app: Final = service.streamable_http_app( + stateless_http=False, + json_response=True, + transport_security=TransportSecuritySettings(enable_dns_rebinding_protection=False), + ) + + async def malformed_method(request: Request, call_next: RequestResponseEndpoint) -> Response: + if request.method == "GET": + return Response(content="data: {}\n\n", media_type="text/event-stream") + if request.method == "POST": + payload: Final = await request.json() + if isinstance(payload, dict) and payload.get("method") == method: + return Response(content="not a JSON-RPC message", media_type="text/event-stream") + return await call_next(request) + + app.add_middleware(BaseHTTPMiddleware, dispatch=malformed_method) + return app + + +@pytest.mark.asyncio +async def test_persistent_session_keeps_upstream_state_across_tool_calls() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + client: Final = _StatefulUpstreamClient(app, server_url="http://upstream/mcp", transport_type=MCPTransport.http) + per_call: Final = await client.call_tool(CallToolRequestParams(name="select_project", arguments={"name": "a"})) + assert per_call.is_error is False + fresh: Final = await client.call_tool(CallToolRequestParams(name="create_feature", arguments={"title": "b"})) + assert fresh.is_error is True, "a fresh upstream session per call must not see the earlier selection" + + session: Final = client.open_persistent_session() + try: + selected: Final = await client.call_tool( + CallToolRequestParams(name="select_project", arguments={"name": "a"}), persistent_session=session + ) + created: Final = await client.call_tool( + CallToolRequestParams(name="create_feature", arguments={"title": "b"}), persistent_session=session + ) + finally: + session.close() + assert selected.is_error is False + assert created.is_error is False + assert created.content[0].text == "a/b" + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed + + +def _client_with_session( + app: Starlette, + admission: Callable[[], AbstractAsyncContextManager[None]] | None = None, +) -> tuple[_StatefulUpstreamClient, PersistentMCPSession]: + client: Final = _StatefulUpstreamClient(app, server_url="http://upstream/mcp", transport_type=MCPTransport.http) + return client, client.open_persistent_session(admission) + + +@pytest.mark.asyncio +async def test_cancelling_one_waiter_for_session_readiness_does_not_cancel_other_waiters() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + initialize_started: Final = asyncio.Event() + release_initialize: Final = asyncio.Event() + + async def gated_server(scope: Scope, receive: Receive, send: Send) -> None: + async def gated_send(message: Message) -> None: + if message["type"] == "http.response.body" and not initialize_started.is_set(): + initialize_started.set() + await release_initialize.wait() + await send(message) + + await app(scope, receive, gated_send) + + gated_app: Final[ASGIApp] = gated_server + client: Final = _StatefulUpstreamClient( + gated_app, server_url="http://upstream/mcp", transport_type=MCPTransport.http + ) + session: Final = client.open_persistent_session() + + async def operation(_: ClientSession) -> str: + return "ready" + + cancelled_waiter: Final = asyncio.create_task(session.run(operation)) + remaining_waiter: Final = asyncio.create_task(session.run(operation)) + try: + await asyncio.wait_for(initialize_started.wait(), 5) + await asyncio.sleep(0) + cancelled_waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await cancelled_waiter + release_initialize.set() + assert await asyncio.wait_for(remaining_waiter, 5) == "ready" + assert not session.closed + finally: + release_initialize.set() + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_survives_a_caller_timeout_on_one_operation() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + client, session = _client_with_session(app) + try: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for( + client.call_tool( + CallToolRequestParams(name="select_project", arguments={"name": "a"}), + persistent_session=session, + ), + timeout=0, + ) + selected: Final = await client.call_tool( + CallToolRequestParams(name="select_project", arguments={"name": "c"}), persistent_session=session + ) + created: Final = await client.call_tool( + CallToolRequestParams(name="create_feature", arguments={"title": "d"}), persistent_session=session + ) + finally: + session.close() + assert selected.is_error is False, "the session must outlive a timed out call" + assert created.content[0].text == "c/d" + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_survives_recoverable_mcp_errors_and_closes_on_connection_closed() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + client, session = _client_with_session(app) + + async def invalid_operation(_: ClientSession) -> str: + raise MCPError(code=-32602, message="invalid params") + + try: + with pytest.raises(MCPError): + await session.run(invalid_operation) + assert not session.closed + selected: Final = await client.call_tool( + CallToolRequestParams(name="select_project", arguments={"name": "recoverable"}), + persistent_session=session, + ) + created: Final = await client.call_tool( + CallToolRequestParams(name="create_feature", arguments={"title": "still-open"}), + persistent_session=session, + ) + finally: + session.close() + assert selected.is_error is False + assert created.content[0].text == "recoverable/still-open" + await asyncio.wait_for(session.wait_closed(), 5) + + connection_app: Final = _stateful_upstream() + async with connection_app.router.lifespan_context(connection_app): + _, session = _client_with_session(connection_app) + started: Final = asyncio.Event() + release: Final = asyncio.Event() + + async def disconnected_operation(_: ClientSession) -> str: + started.set() + await release.wait() + raise MCPError(code=CONNECTION_CLOSED, message="upstream connection closed") + + async def pending_operation(_: ClientSession) -> str: + return "unused" + + active: Final = asyncio.create_task(session.run(disconnected_operation)) + await asyncio.wait_for(started.wait(), 5) + pending: Final = asyncio.create_task(session.run(pending_operation)) + await asyncio.sleep(0) + release.set() + with pytest.raises(MCPError) as caught: + await asyncio.wait_for(active, 5) + assert caught.value.error.code == CONNECTION_CLOSED + with pytest.raises((MCPError, UpstreamSessionClosedError)): + await asyncio.wait_for(pending, 5) + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed + + +@pytest.mark.asyncio +async def test_ended_persistent_session_is_not_reusable_during_transport_cleanup() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + cleanup_started: Final = asyncio.Event() + release_cleanup: Final = asyncio.Event() + client: Final = _CleanupHoldingStatefulUpstreamClient( + app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + cleanup_started=cleanup_started, + release_cleanup=release_cleanup, + ) + session: Final = client.open_persistent_session() + + async def disconnected_operation(_: ClientSession) -> str: + raise MCPError(code=CONNECTION_CLOSED, message="upstream connection closed") + + try: + with pytest.raises(MCPError) as caught: + await asyncio.wait_for(session.run(disconnected_operation), 5) + assert caught.value.error.code == CONNECTION_CLOSED + await asyncio.wait_for(cleanup_started.wait(), 5) + + assert not session.reusable + assert not session.closed + session.retire() + + release_cleanup.set() + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed + finally: + release_cleanup.set() + if not session.closed: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_tool_call_preserves_upstream_http_status_error() -> None: + app: Final = _upstream_with_status_for_method("tools/call", 401) + async with app.router.lifespan_context(app): + client: Final = _StatefulUpstreamClient( + app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + ) + session: Final = client.open_persistent_session() + try: + with pytest.raises(httpx2.HTTPStatusError) as caught: + await client.call_tool( + CallToolRequestParams(name="protected_tool", arguments={}), + raise_on_error=True, + persistent_session=session, + ) + assert caught.value.response.status_code == 401 + assert not session.reusable + finally: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_remaps_stream_error_over_mcp_error() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + client: Final = _StatefulUpstreamClient( + app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + ) + session: Final = client.open_persistent_session() + request: Final = httpx2.Request("POST", "http://upstream/mcp") + response: Final = httpx2.Response(401, request=request, headers={"WWW-Authenticate": "Bearer"}) + http_error: Final = httpx2.HTTPStatusError("401 Unauthorized", request=request, response=response) + + async def operation(_: ClientSession) -> str: + stream_error: Final[asyncio.Future[Exception] | None] = session._stream_error + assert stream_error is not None + stream_error.set_result(http_error) + raise MCPError(code=-32603, message="response stream closed") + + try: + with pytest.raises(httpx2.HTTPStatusError) as caught: + await session.run(operation) + assert caught.value is http_error + assert not session.reusable + finally: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_ends_when_a_malformed_response_is_remapped() -> None: + app: Final = _upstream_with_malformed_response_for_method("tools/call") + async with app.router.lifespan_context(app): + client: Final = _StatefulUpstreamClient( + app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + ) + session: Final = client.open_persistent_session() + try: + + async def ready(_: ClientSession) -> str: + return "ready" + + assert await asyncio.wait_for(session.run(ready), 5) == "ready" + stream_error: Final[asyncio.Future[Exception] | None] = session._stream_error + assert stream_error is not None + recorded_stream_error: Final = await asyncio.wait_for(asyncio.shield(stream_error), 5) + assert isinstance(recorded_stream_error, ValidationError) + + with pytest.raises(ValidationError) as caught: + await client.call_tool( + CallToolRequestParams(name="malformed_tool", arguments={}), + raise_on_error=True, + persistent_session=session, + ) + assert caught.value is recorded_stream_error + assert not session.reusable + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed + finally: + if not session.closed: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_failed_persistent_initialize_is_not_reusable_during_transport_cleanup() -> None: + app: Final = _upstream_with_status_for_method("initialize", 500) + async with app.router.lifespan_context(app): + cleanup_started: Final = asyncio.Event() + release_cleanup: Final = asyncio.Event() + client: Final = _CleanupHoldingStatefulUpstreamClient( + app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + cleanup_started=cleanup_started, + release_cleanup=release_cleanup, + ) + session: Final = client.open_persistent_session() + pending: Final = asyncio.create_task(session.run(lambda _: "ready")) + try: + await asyncio.wait_for(cleanup_started.wait(), 5) + assert not session.reusable + assert not session.closed + release_cleanup.set() + with pytest.raises(httpx2.HTTPStatusError): + await asyncio.wait_for(pending, 5) + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed + finally: + release_cleanup.set() + if not session.closed: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_routes_elicitation_to_the_current_clients_callback() -> None: + service: Final = UpstreamServer("elicitation") + + @service.tool() + async def ask_for_label(ctx: Context) -> str: + response: Final = await ctx.elicit("What label should be used?", _ElicitationAnswer) + return response.data.label if response.data is not None else response.action + + async def first_callback(context: ClientRequestContext, params: ElicitRequestParams) -> ElicitResult: + return ElicitResult(action="accept", content={"label": "first"}) + + async def second_callback(context: ClientRequestContext, params: ElicitRequestParams) -> ElicitResult: + return ElicitResult(action="accept", content={"label": "second"}) + + first_client: Final = _InProcessUpstreamClient(service, elicitation_callback=first_callback) + second_client: Final = _InProcessUpstreamClient(service, elicitation_callback=second_callback) + session: Final = first_client.open_persistent_session() + try: + first: Final = await first_client.call_tool( + CallToolRequestParams(name="ask_for_label", arguments={}), persistent_session=session + ) + second: Final = await second_client.call_tool( + CallToolRequestParams(name="ask_for_label", arguments={}), persistent_session=session + ) + finally: + session.close() + assert first.is_error is False and first.content[0].text == "first" + assert second.is_error is False and second.content[0].text == "second" + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_without_elicitation_callback_uses_sdk_default() -> None: + service: Final = UpstreamServer("elicitation default") + + @service.tool() + async def ask_for_label(ctx: Context) -> str: + response: Final = await ctx.elicit("What label should be used?", _ElicitationAnswer) + return response.data.label if response.data is not None else response.action + + client: Final = _InProcessUpstreamClient(service) + per_call: Final = await client.call_tool(CallToolRequestParams(name="ask_for_label", arguments={})) + session: Final = client.open_persistent_session() + try: + persistent: Final = await client.call_tool( + CallToolRequestParams(name="ask_for_label", arguments={}), persistent_session=session + ) + finally: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + assert persistent.model_dump(mode="json") == per_call.model_dump(mode="json") + assert "elicitation callback is unavailable" not in str(persistent.model_dump(mode="json")) + + +@pytest.mark.asyncio +async def test_cancelled_queued_operation_does_not_run_and_session_accepts_a_later_operation() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + _, session = _client_with_session(app) + first_started: Final = asyncio.Event() + release_first: Final = asyncio.Event() + cancelled_operation_ran: Final = asyncio.Event() + + async def first_operation(_: ClientSession) -> str: + first_started.set() + await release_first.wait() + return "first" + + async def cancelled_operation(_: ClientSession) -> str: + cancelled_operation_ran.set() + return "cancelled" + + async def third_operation(_: ClientSession) -> str: + return "third" + + first_task: Final = asyncio.create_task(session.run(first_operation)) + try: + await asyncio.wait_for(first_started.wait(), 5) + second_task: Final = asyncio.create_task(session.run(cancelled_operation)) + await asyncio.sleep(0) + assert session._queue.qsize() == 1 + second_task.cancel() + with pytest.raises(asyncio.CancelledError): + await second_task + third_task: Final = asyncio.create_task(session.run(third_operation)) + await asyncio.sleep(0) + assert session._queue.qsize() == 2 + release_first.set() + assert await asyncio.wait_for(first_task, 5) == "first" + assert await asyncio.wait_for(third_task, 5) == "third" + assert not cancelled_operation_ran.is_set() + assert not session.closed + finally: + release_first.set() + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_retiring_persistent_session_drains_in_flight_and_queued_operations() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + _, session = _client_with_session(app) + first_started: Final = asyncio.Event() + release_first: Final = asyncio.Event() + + async def first_operation(_: ClientSession) -> str: + first_started.set() + await release_first.wait() + return "first" + + async def queued_operation(_: ClientSession) -> str: + return "queued" + + first_waiter: Final = asyncio.create_task(session.run(first_operation)) + try: + await asyncio.wait_for(first_started.wait(), 5) + queued_waiter: Final = asyncio.create_task(session.run(queued_operation)) + await asyncio.sleep(0) + assert session._queue.qsize() == 1 + session.retire() + release_first.set() + + results: Final = await asyncio.wait_for(asyncio.gather(first_waiter, queued_waiter), 5) + assert results == ["first", "queued"] + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed + finally: + release_first.set() + if not session.closed: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_timed_out_persistent_operation_waiting_for_admission_is_skipped() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + admission_slots: Final = asyncio.Semaphore(1) + admission_requested: Final = asyncio.Event() + + def admission() -> AbstractAsyncContextManager[None]: + admission_requested.set() + return admission_slots + + _, session = _client_with_session(app, admission) + operation_ran: Final = asyncio.Event() + + async def warmup(_: ClientSession) -> str: + return "ready" + + async def timed_out_operation(_: ClientSession) -> str: + operation_ran.set() + return "expired" + + async def later_operation(_: ClientSession) -> str: + return "later" + + try: + assert await session.run(warmup) == "ready" + admission_requested.clear() + async with admission_slots: + timed_out_call: Final = asyncio.create_task(session.run(timed_out_operation)) + await asyncio.wait_for(admission_requested.wait(), 1) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(timed_out_call, 0.05) + assert not operation_ran.is_set() + + later_result: Final = await asyncio.wait_for(session.run(later_operation), 1) + + assert later_result == "later" + assert session.reusable + assert not operation_ran.is_set() + finally: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_holds_admission_until_initialize_completes() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + initialize_seen: Final = asyncio.Event() + admission_slots: Final = asyncio.Semaphore(1) + + async def observing_app(scope: Scope, receive: Receive, send: Send) -> None: + async def observe_send(message: Message) -> None: + if message["type"] == "http.response.body": + initialize_seen.set() + await send(message) + + await app(scope, receive, observe_send) + + client: Final = _StatefulUpstreamClient( + observing_app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + ) + session: Final = client.open_persistent_session(lambda: admission_slots) + + async def operation(_: ClientSession) -> str: + return "ready" + + run_task: Final = asyncio.create_task(session.run(operation)) + try: + async with admission_slots: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(initialize_seen.wait(), 0.05) + assert not session._ready.done() + assert not run_task.done() + + result: Final = await asyncio.wait_for(run_task, 5) + assert result == "ready" + assert initialize_seen.is_set() + finally: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + await asyncio.gather(run_task, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_persistent_session_releases_admission_when_connect_fails() -> None: + async def failing_app(_: Scope, __: Receive, ___: Send) -> None: + raise httpx2.ConnectError("connect failed") + + client: Final = _StatefulUpstreamClient( + failing_app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + ) + admission_slots: Final = asyncio.Semaphore(1) + session: Final = client.open_persistent_session(lambda: admission_slots) + + async def operation(_: ClientSession) -> str: + return "unreachable" + + try: + with pytest.raises(httpx2.ConnectError, match="connect failed"): + await asyncio.wait_for(session.run(operation), 1) + await asyncio.wait_for(session.wait_closed(), 1) + + await asyncio.wait_for(admission_slots.acquire(), 0.1) + admission_slots.release() + finally: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_failed_connect_is_not_reusable_during_admission_teardown() -> None: + connect_failed: Final = asyncio.Event() + admission_exit_started: Final = asyncio.Event() + release_admission_exit: Final = asyncio.Event() + + async def failing_app(_: Scope, __: Receive, ___: Send) -> None: + connect_failed.set() + raise httpx2.ConnectError("connect failed") + + @asynccontextmanager + async def blocking_admission() -> AsyncIterator[None]: + try: + yield + finally: + admission_exit_started.set() + await release_admission_exit.wait() + + client: Final = _StatefulUpstreamClient( + failing_app, + server_url="http://upstream/mcp", + transport_type=MCPTransport.http, + ) + session: Final = client.open_persistent_session(blocking_admission) + + async def operation(_: ClientSession) -> str: + return "unreachable" + + run_task: Final = asyncio.create_task(session.run(operation)) + try: + await asyncio.wait_for(connect_failed.wait(), 1) + await asyncio.wait_for(admission_exit_started.wait(), 1) + + assert not run_task.done() + assert not session.reusable + assert not session.closed + + release_admission_exit.set() + with pytest.raises(httpx2.ConnectError, match="connect failed"): + await asyncio.wait_for(run_task, 1) + await asyncio.wait_for(session.wait_closed(), 1) + assert session.closed + finally: + release_admission_exit.set() + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + await asyncio.gather(run_task, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_persistent_session_admission_is_acquired_only_when_operation_runs() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + admission_slots: Final = asyncio.Semaphore(2) + _, session = _client_with_session(app, lambda: admission_slots) + first_started: Final = asyncio.Event() + release_first: Final = asyncio.Event() + operation_order: Final[list[str]] = [] + + async def first_operation(_: ClientSession) -> str: + operation_order.append("first") + first_started.set() + await release_first.wait() + return "first" + + async def second_operation(_: ClientSession) -> str: + operation_order.append("second") + return "second" + + async def third_operation(_: ClientSession) -> str: + operation_order.append("third") + return "third" + + first_task: Final = asyncio.create_task(session.run(first_operation)) + second_task: Final = asyncio.create_task(session.run(second_operation)) + third_task: Final = asyncio.create_task(session.run(third_operation)) + try: + await asyncio.wait_for(first_started.wait(), 5) + await asyncio.sleep(0) + assert session._queue.qsize() == 2 + await asyncio.wait_for(admission_slots.acquire(), 0.1) + try: + assert admission_slots.locked() + finally: + admission_slots.release() + assert operation_order == ["first"] + + release_first.set() + results: Final = await asyncio.wait_for( + asyncio.gather(first_task, second_task, third_task), + 5, + ) + + assert results == ["first", "second", "third"] + assert operation_order == ["first", "second", "third"] + finally: + release_first.set() + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + await asyncio.gather(first_task, second_task, third_task, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_retiring_idle_persistent_session_closes_and_rejects_new_operations() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + _, session = _client_with_session(app) + + async def operation(_: ClientSession) -> str: + return "ready" + + try: + assert await session.run(operation) == "ready" + session.retire() + await asyncio.wait_for(session.wait_closed(), 1) + + assert session.closed + with pytest.raises(UpstreamSessionClosedError): + await session.run(operation) + finally: + if not session.closed: + session.close() + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_closing_persistent_session_mid_operation_fails_the_waiter_instead_of_hanging() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + _, session = _client_with_session(app) + started: Final = asyncio.Event() + + async def slow_operation(_: object) -> str: + started.set() + await asyncio.sleep(30) + return "never" + + waiter: Final = asyncio.ensure_future(session.run(slow_operation)) + await asyncio.wait_for(started.wait(), 5) + session.close() + with pytest.raises(RuntimeError, match="upstream MCP session closed"): + await asyncio.wait_for(waiter, 5) + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_reports_an_upstream_cancellation_as_a_runtime_error_not_a_cancelled_caller() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + client, session = _client_with_session(app) + + async def cancelled_upstream(_: object) -> str: + raise asyncio.CancelledError() + + try: + with pytest.raises(RuntimeError, match="cancelled"): + await asyncio.wait_for(session.run(cancelled_upstream), 5) + selected: Final = await client.call_tool( + CallToolRequestParams(name="select_project", arguments={"name": "e"}), persistent_session=session + ) + finally: + session.close() + assert selected.is_error is False, "an upstream cancellation must not be mistaken for a cancelled caller" + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_closing_persistent_session_fails_a_caller_blocked_on_a_full_queue_instead_of_hanging() -> None: + from litellm.experimental_mcp_client.client import _MAX_PENDING_OPERATIONS + + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + _, session = _client_with_session(app) + started: Final = asyncio.Event() + + async def slow_operation(_: object) -> str: + started.set() + await asyncio.sleep(30) + return "never" + + waiters: Final = tuple( + asyncio.ensure_future(session.run(slow_operation)) for _ in range(_MAX_PENDING_OPERATIONS + 2) + ) + await asyncio.wait_for(started.wait(), 5) + await asyncio.sleep(0) + session.close() + outcomes: Final = await asyncio.wait_for(asyncio.gather(*waiters, return_exceptions=True), 5) + assert all(isinstance(outcome, RuntimeError) for outcome in outcomes), outcomes + await asyncio.wait_for(session.wait_closed(), 5) + + +@pytest.mark.asyncio +async def test_persistent_session_ends_after_a_broken_stream_so_the_next_call_gets_a_fresh_one() -> None: + app: Final = _stateful_upstream() + async with app.router.lifespan_context(app): + _, session = _client_with_session(app) + + async def broken_stream(_: object) -> str: + raise anyio.BrokenResourceError() + + with pytest.raises(anyio.BrokenResourceError): + await asyncio.wait_for(session.run(broken_stream), 5) + await asyncio.wait_for(session.wait_closed(), 5) + assert session.closed, "a dead transport must end the session instead of being reused" diff --git a/tests/unit/integrations/azure_storage/test_azure_storage.py b/tests/unit/integrations/azure_storage/test_azure_storage.py index 01b17edd2cf..0f0ce24ff84 100644 --- a/tests/unit/integrations/azure_storage/test_azure_storage.py +++ b/tests/unit/integrations/azure_storage/test_azure_storage.py @@ -4,18 +4,20 @@ import json import re import sys import threading +from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm.constants import _DEFAULT_TTL_FOR_HTTPX_CLIENTS +from litellm.integrations.azure_storage import azure_storage as azure_storage_module from litellm.integrations.azure_storage.azure_storage import ( AzureBlobStorageLogger, _cached_credential_chain_token_provider, adls_safe_file_name, ) from litellm.types.secret_managers.get_azure_ad_token_provider import AzureCredentialType -from litellm.types.utils import StandardLoggingPayload +from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload @pytest.fixture @@ -336,11 +338,8 @@ async def test_account_key_auth_never_requests_a_token(workload_identity_env_var file_client.create_file = AsyncMock() file_client.append_data = AsyncMock() file_client.flush_data = AsyncMock() - directory_client = MagicMock() - directory_client.exists = AsyncMock(return_value=True) - directory_client.get_file_client = MagicMock(return_value=file_client) file_system_client = MagicMock() - file_system_client.get_directory_client = MagicMock(return_value=directory_client) + file_system_client.get_file_client = MagicMock(return_value=file_client) service_client = MagicMock() service_client.get_file_system_client = MagicMock(return_value=file_system_client) fake_aio_module = MagicMock() @@ -523,11 +522,8 @@ async def test_account_key_upload_names_the_file_adls_safe_and_keeps_the_origina file_client.create_file = AsyncMock() file_client.append_data = AsyncMock() file_client.flush_data = AsyncMock() - directory_client = MagicMock() - directory_client.exists = AsyncMock(return_value=True) - directory_client.get_file_client = MagicMock(return_value=file_client) file_system_client = MagicMock() - file_system_client.get_directory_client = MagicMock(return_value=directory_client) + file_system_client.get_file_client = MagicMock(return_value=file_client) service_client = MagicMock() service_client.get_file_system_client = MagicMock(return_value=file_system_client) fake_aio_module = MagicMock() @@ -537,7 +533,9 @@ async def test_account_key_upload_names_the_file_adls_safe_and_keeps_the_origina logger = AzureBlobStorageLogger() await logger.async_upload_payload_to_azure_blob_storage({"id": "resp_YWJjZA=="}) - directory_client.get_file_client.assert_called_once_with("resp_YWJjZA.json") + file_system_client.get_file_client.assert_called_once_with( + f"{datetime.now().strftime('%Y-%m-%d')}/resp_YWJjZA.json" + ) body = json.loads(file_client.append_data.call_args.kwargs["data"]) assert body["id"] == "resp_YWJjZA==", "the stored payload must keep the original id byte for byte" @@ -569,3 +567,84 @@ async def test_entra_upload_names_the_file_adls_safe_and_keeps_the_original_id(m ), f"the Entra path must be the rewritten name, got {put_call_args[0][0]!r}" append_call = mock_http_client.patch.call_args_list[0] assert "resp_YWJjZA==" in append_call[1]["data"], "the stored payload must keep the original id byte for byte" + + +class _FrozenDatetime(datetime): + @classmethod + def now(cls, tz=None): + instant = datetime(2026, 9, 30, 3, 5, 9, tzinfo=timezone.utc) + if tz is None: + return instant.astimezone(timezone(timedelta(hours=-7))).replace(tzinfo=None) + return instant.astimezone(tz) + + +def _key_created_audit_log(audit_log_id: str) -> StandardAuditLogPayload: + return StandardAuditLogPayload( + id=audit_log_id, + updated_at="2026-09-30T03:05:09+00:00", + changed_by="admin-user", + changed_by_api_key="hashed-admin-key", + action="created", + table_name="LiteLLM_VerificationToken", + object_id="hashed-new-key", + before_value=None, + updated_values=json.dumps({"key_alias": "audit-key"}), + ) + + +@pytest.mark.asyncio +async def test_audit_log_is_uploaded_to_a_dated_audit_logs_path_with_a_service_principal(mock_env_vars, monkeypatch): + monkeypatch.setattr(azure_storage_module, "datetime", _FrozenDatetime) + audit_log = _key_created_audit_log("audit-id-sp") + with patch( # test-quality-ok: REST client is created inside the method; assert emitted requests + "litellm.integrations.azure_storage.azure_storage.get_async_httpx_client" + ) as mock_get_client: + mock_http_client = AsyncMock() + mock_http_client.put.return_value = MagicMock() + mock_http_client.patch.return_value = MagicMock() + mock_get_client.return_value = mock_http_client + + logger = AzureBlobStorageLogger() + logger.azure_auth_token = "sp-token" + logger.token_expiry = datetime(2026, 9, 30, 9, 30) + await logger.async_log_audit_log_event(audit_log) + + file_url = ( + "https://test-account.dfs.core.windows.net/test-container/audit_logs/2026-09-30/03-05-09_audit-id-sp.json" + ) + assert [call.args[0] for call in mock_http_client.put.call_args_list] == [f"{file_url}?resource=file"] + append_call, flush_call = mock_http_client.patch.call_args_list + assert append_call.args[0] == f"{file_url}?action=append&position=0" + assert json.loads(append_call.kwargs["data"]) == audit_log + assert flush_call.args[0] == f"{file_url}?action=flush&position={len(append_call.kwargs['data'].encode())}" + assert logger.log_queue == [], "audit logs must not be mixed into the request log queue" + + +@pytest.mark.asyncio +async def test_audit_log_is_uploaded_to_a_dated_audit_logs_path_with_an_account_key(mock_env_vars, monkeypatch): + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_KEY", "dGVzdC1rZXk=") + monkeypatch.setattr(azure_storage_module, "datetime", _FrozenDatetime) + audit_log = _key_created_audit_log("audit-id-key") + + file_client = MagicMock() + file_client.create_file = AsyncMock() + file_client.append_data = AsyncMock() + file_client.flush_data = AsyncMock() + file_system_client = MagicMock() + file_system_client.get_file_client = MagicMock(return_value=file_client) + service_client = MagicMock() + service_client.get_file_system_client = MagicMock(return_value=file_system_client) + fake_aio_module = MagicMock() + fake_aio_module.DataLakeServiceClient = MagicMock(return_value=service_client) + + with patch.dict(sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module}): + logger = AzureBlobStorageLogger() + await logger.async_log_audit_log_event(audit_log) + + service_client.get_file_system_client.assert_called_once_with(file_system="test-container") + file_system_client.get_file_client.assert_called_once_with("audit_logs/2026-09-30/03-05-09_audit-id-key.json") + file_client.create_file.assert_awaited_once_with() + uploaded = file_client.append_data.await_args.kwargs["data"] + assert json.loads(uploaded) == audit_log + assert file_client.append_data.await_args.kwargs == {"data": uploaded, "offset": 0, "length": len(uploaded)} + file_client.flush_data.assert_awaited_once_with(position=len(uploaded), offset=0) diff --git a/tests/unit/integrations/otel/test_langfuse_logger.py b/tests/unit/integrations/otel/test_langfuse_logger.py index aca9dcc8a5e..ff2060d91c2 100644 --- a/tests/unit/integrations/otel/test_langfuse_logger.py +++ b/tests/unit/integrations/otel/test_langfuse_logger.py @@ -16,6 +16,7 @@ import litellm # noqa: E402 from litellm.caching.dual_cache import DualCache # noqa: E402 from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger # noqa: E402 from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402 +from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402 from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402 from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402 from litellm.integrations.otel.plumbing import providers # noqa: E402 @@ -155,6 +156,25 @@ def test_chat_request_stamps_root_observation_input_and_output(): assert [(turn["role"], turn["content"]) for turn in output] == [("assistant", "pong")] +def test_content_on_destination_stamps_langfuse_root_io_when_global_capture_is_off() -> None: + logger, exporter = _logger(capture="no_content") + destination: Final = OtelDestination( + endpoint="https://team.example.com/api/public/otel", + callback_name="langfuse_otel", + capture_message_content="span_only", + ) + token: Final = otel_context.set_request_destinations((destination,)) + try: + response: Final = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + _run_request(logger, CHAT_DATA, "acompletion", response) + finally: + otel_context.reset_request_destinations(token) + + attrs: Final = _root_attrs(exporter) + assert json.loads(attrs[INPUT_ATTR]) == [{"role": "user", "content": "ping"}] + assert json.loads(attrs[OUTPUT_ATTR]) == [{"role": "assistant", "content": "pong"}] + + def test_responses_request_folds_instructions_into_input_and_stamps_output_items(): logger, exporter = _logger() data = {"model": "gpt-5.4-mini", "instructions": "be terse", "input": "ping"} @@ -461,7 +481,9 @@ def test_a_request_without_trace_controls_stamps_none_of_them(): logger, exporter = _logger() root_attrs, generation_attrs = _run_named_request( - logger, exporter, {"metadata": {"user_api_key_team_id": "t1", "tags": []}, "proxy_server_request": {"headers": {}}} + logger, + exporter, + {"metadata": {"user_api_key_team_id": "t1", "tags": []}, "proxy_server_request": {"headers": {}}}, ) assert set(TRACE_CONTROL_ATTRS).isdisjoint(root_attrs) @@ -472,7 +494,7 @@ def test_a_request_without_trace_controls_stamps_none_of_them(): ("capture", "mappers"), [("no_content", ("genai", "langfuse")), ("span_only", ("genai",))], ) -def test_factory_keeps_the_base_logger_unless_langfuse_content_capture_is_on(capture, mappers): +def test_langfuse_root_io_requires_global_or_destination_capture(capture: str, mappers: tuple[str, ...]) -> None: logger, exporter = _logger(capture=capture, mappers=mappers) _run_request(logger, CHAT_DATA, "acompletion", ModelResponse()) @@ -484,13 +506,13 @@ def test_factory_keeps_the_base_logger_unless_langfuse_content_capture_is_on(cap ("capture", "mappers", "relays_streams"), [ ("span_only", ("genai", "langfuse"), True), - ("no_content", ("genai", "langfuse"), False), + ("no_content", ("genai", "langfuse"), True), ("span_only", ("genai",), False), ], ) -def test_only_langfuse_content_capture_takes_proxy_streams_off_the_fast_path( - monkeypatch, capture, mappers, relays_streams -): +def test_langfuse_mapper_relays_streams_for_per_request_content_capture( + monkeypatch: pytest.MonkeyPatch, capture: str, mappers: tuple[str, ...], relays_streams: bool +) -> None: logger, _ = _logger(capture=capture, mappers=mappers) monkeypatch.setattr(litellm, "callbacks", [logger]) diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index 2d7e0745039..5c9cab88784 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -5,6 +5,7 @@ import contextvars import time from base64 import b64encode from collections.abc import Mapping +from dataclasses import replace from datetime import datetime, timezone from functools import reduce from types import MappingProxyType @@ -13,13 +14,14 @@ from typing import Final import pytest from opentelemetry import trace as trace_api from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace import Status, StatusCode import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.otel import GenAIOperation from litellm.integrations.otel import logger as otel_logger from litellm.integrations.otel.logger import ( OpenTelemetryV2, @@ -27,13 +29,24 @@ from litellm.integrations.otel.logger import ( fan_out_provider, publish_global_otel_v2_provider, ) +from litellm.integrations.otel.mappers import resolve_mappers from litellm.integrations.otel.model.config import ( + CaptureMessageContent, ExporterOwner, ExporterSpec, OpenTelemetryV2Config, is_otel_v2_enabled, ) from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel.model.payloads import ( + LLMCallSpanData, + LLMRequestParams, + LLMUsage, + MCPToolCallSpanData, + RequestIdentity, + ServerInfo, + ToolDefinition, +) from litellm.integrations.otel.plumbing import providers as otel_providers from litellm.integrations.otel.plumbing.context import ( destination_backends, @@ -44,9 +57,11 @@ from litellm.integrations.otel.plumbing.providers import ( TenantFanOutSpanProcessor, _OverriddenBackendFilter, _sink_key, + attach_tenant_fan_out, build_tracer_provider, deliverable_destinations, operator_sink_scopes, + register_exporter_factory, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache, get_tracer from litellm.integrations.otel.presets.arize import arize_preset @@ -55,7 +70,7 @@ from litellm.integrations.otel.presets.destinations import ( destination_for, ) from litellm.integrations.otel.presets.langfuse import langfuse_preset -from litellm.proxy._types import AddTeamCallback, UserAPIKeyAuth +from litellm.proxy._types import AddTeamCallback, TeamCallbackMetadata, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import ( convert_key_logging_metadata_to_callback, resolve_tenant_otel_destinations, @@ -4228,6 +4243,569 @@ class TestTenantHostSsrfGuard: assert sum("provider_url_destination_allowed_hosts" in record.message for record in caplog.records) == 1 +_ALL_MAPPERS: Final = ("genai", "legacy", "openinference", "langfuse", "weave", "langtrace") +_SECRET: Final = "SECRET-MARKER" +_MODEL_CALL_WITH_CONTENT: Final = LLMCallSpanData( + operation=GenAIOperation.CHAT, + provider="openai", + request_model="gpt-4o", + response_model="gpt-4o-2024", + response_id="resp_1", + request_params=LLMRequestParams(temperature=0.5, max_tokens=128), + usage=LLMUsage(input_tokens=12, output_tokens=8, total_tokens=20), + finish_reasons=("tool_calls",), + error=None, + response_cost=0.001, + server=ServerInfo("api.openai.com", 443), + identity=RequestIdentity(call_id="c1", team_id="t1"), + tools=(ToolDefinition(name="lookup", description="Find a city", parameters_json='{"type":"object"}'),), + messages_in=( + {"role": "system", "content": f"system {_SECRET}"}, + {"role": "user", "content": f"prompt {_SECRET}"}, + ), + choices_out=( + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": f"answer {_SECRET}", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": f'{{"city": "{_SECRET}"}}'}, + } + ], + }, + }, + ), +) +_MODEL_CALL_WITHOUT_CONTENT: Final = replace(_MODEL_CALL_WITH_CONTENT, messages_in=(), choices_out=()) +_TOOL_CALL_WITH_CONTENT: Final = MCPToolCallSpanData( + operation=GenAIOperation.EXECUTE_TOOL, + method="tools/call", + tool_name="lookup", + server_name="maps", + server_address="maps.local", + server_port=443, + session_id="s1", + arguments_json=f'{{"city": "{_SECRET}"}}', + result_json=f'{{"weather": "{_SECRET}"}}', + error=None, + response_cost=None, + identity=RequestIdentity(call_id="c2", team_id="t1"), +) +_TOOL_CALL_WITHOUT_CONTENT: Final = replace(_TOOL_CALL_WITH_CONTENT, arguments_json=None, result_json=None) + +NO_CONTENT_DEST = OtelDestination( + endpoint="http://team-a.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic YQ=="}), + callback_name="langfuse_otel", + capture_message_content="no_content", +) +INHERITING_DEST = OtelDestination( + endpoint="http://team-b.local/v1", + headers=MappingProxyType({"api_key": "k", "arize-space-id": "s"}), + callback_name="arize", +) + + +def mapped(data: LLMCallSpanData | MCPToolCallSpanData) -> Mapping[str, object]: + """What the configured mappers write on the span, the way the logger stamps it.""" + return MappingProxyType(reduce(lambda acc, mapper: {**acc, **mapper.map(data)}, resolve_mappers(_ALL_MAPPERS), {})) + + +def recorded(attributes: Mapping[str, object]) -> dict[str, object]: + """``attributes`` the way the SDK records them, sequences frozen into tuples.""" + return {key: tuple(value) if isinstance(value, list) else value for key, value in attributes.items()} + + +def model_call_tree(provider: TracerProvider, attributes: Mapping[str, object]) -> None: + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("chat gpt-4o") as llm: + llm.set_attributes(attributes) + llm.add_event("gen_ai.content.first_chunk", {"gen_ai.response.model": "gpt-4o-2024"}) + + +def by_name(exporter: InMemorySpanExporter) -> dict[str, ReadableSpan]: + return {span.name: span for span in exporter.get_finished_spans()} + + +def carries_content(span: ReadableSpan) -> bool: + return any(_SECRET in str(value) for value in span.attributes.values()) + + +class TestCaptureMessageContent: + """A destination's explicit capture mode overrides the global default.""" + + @staticmethod + def _fan_out( + operator: InMemorySpanExporter, + exporters: Mapping[str, InMemorySpanExporter], + default_capture: CaptureMessageContent = CaptureMessageContent.SPAN_ONLY, + ) -> TracerProvider: + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator)) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda d: SimpleSpanProcessor(exporters[d.endpoint]), + default_capture=default_capture, + ) + ) + return provider + + @staticmethod + def _run( + provider: TracerProvider, destinations: tuple[OtelDestination, ...], attributes: Mapping[str, object] + ) -> None: + def run() -> None: + set_request_destinations(destinations) + model_call_tree(provider, attributes) + + in_fresh_context(run) + + @pytest.mark.parametrize( + ("setting", "content_exported"), + [ + (None, True), + ("no_content", False), + ("span_only", True), + ("event_only", False), + ("span_and_event", True), + ], + ) + def test_a_destination_setting_controls_its_copy_and_omission_uses_the_global_default( + self, setting: str | None, content_exported: bool + ) -> None: + attributes: Final = mapped(_MODEL_CALL_WITH_CONTENT) + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + capture_message_content=setting, + ) + operator, tenant = InMemorySpanExporter(), InMemorySpanExporter() + + self._run(self._fan_out(operator, {destination.endpoint: tenant}), (destination,), attributes) + + exported: Final = by_name(tenant)["chat gpt-4o"] + assert carries_content(exported) is content_exported + assert carries_content(by_name(operator)["chat gpt-4o"]) + assert exported.attributes["gen_ai.request.model"] == "gpt-4o" + assert exported.attributes["gen_ai.usage.input_tokens"] == 12 + assert exported.attributes["gen_ai.usage.output_tokens"] == 8 + + def test_a_span_only_team_lifts_global_no_content_only_for_its_destination(self) -> None: + operator_exporter, team_exporter, sibling_exporter = ( + InMemorySpanExporter(), + InMemorySpanExporter(), + InMemorySpanExporter(), + ) + kind: Final = "lit8244_global_no_content_capture" + register_exporter_factory(kind, lambda _spec: operator_exporter) + config: Final = OpenTelemetryV2Config( + capture_message_content=CaptureMessageContent.NO_CONTENT, + exporters=[ExporterSpec(kind=kind)], + ) + provider: Final = build_tracer_provider(config, use_simple_processor=True, tenant_overrides=True) + team: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + capture_message_content="span_only", + ) + sibling: Final = INHERITING_DEST + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda destination: SimpleSpanProcessor( + team_exporter if destination.endpoint == team.endpoint else sibling_exporter + ), + default_capture=config.capture_message_content, + ) + ) + + self._run( + provider, + (team, sibling), + mapped(_MODEL_CALL_WITH_CONTENT), + ) + + assert carries_content(by_name(team_exporter)["chat gpt-4o"]) + assert not any(carries_content(span) for span in operator_exporter.get_finished_spans()) + assert not any(carries_content(span) for span in sibling_exporter.get_finished_spans()) + + @pytest.mark.parametrize( + ("with_content", "without_content"), + [ + (_MODEL_CALL_WITH_CONTENT, _MODEL_CALL_WITHOUT_CONTENT), + (_TOOL_CALL_WITH_CONTENT, _TOOL_CALL_WITHOUT_CONTENT), + ], + ids=["model_call", "mcp_tool_call"], + ) + def test_no_content_removes_exactly_what_global_capture_adds_in_every_mapper_vocabulary( + self, + with_content: LLMCallSpanData | MCPToolCallSpanData, + without_content: LLMCallSpanData | MCPToolCallSpanData, + ) -> None: + captured, uncaptured = mapped(with_content), mapped(without_content) + assert frozenset(captured) - frozenset(uncaptured), "the fixture must exercise captured content" + operator, tenant = InMemorySpanExporter(), InMemorySpanExporter() + + self._run(self._fan_out(operator, {NO_CONTENT_DEST.endpoint: tenant}), (NO_CONTENT_DEST,), captured) + + exported: Final = by_name(tenant)["chat gpt-4o"] + assert dict(exported.attributes) == recorded(uncaptured), "only content goes, every other attribute stays" + assert not carries_content(exported) + + def test_restricting_one_destination_leaves_the_original_span_and_the_other_destination_alone(self) -> None: + attributes: Final = mapped(_MODEL_CALL_WITH_CONTENT) + operator, restricted, inheriting = InMemorySpanExporter(), InMemorySpanExporter(), InMemorySpanExporter() + exporters: Final = {NO_CONTENT_DEST.endpoint: restricted, INHERITING_DEST.endpoint: inheriting} + + self._run(self._fan_out(operator, exporters), (NO_CONTENT_DEST, INHERITING_DEST), attributes) + + original: Final = by_name(operator) + kept: Final = by_name(inheriting) + stripped: Final = by_name(restricted) + assert dict(original["chat gpt-4o"].attributes) == recorded(attributes) + assert dict(kept["chat gpt-4o"].attributes) == recorded(attributes) + assert not carries_content(stripped["chat gpt-4o"]) + for copy in (kept, stripped): + for name, span in copy.items(): + assert span.context == original[name].context, "same trace id and span id" + assert span.parent == original[name].parent, "same place in the tree" + assert [e.name for e in copy["chat gpt-4o"].events] == [e.name for e in original["chat gpt-4o"].events] + + def test_an_omitted_setting_and_span_only_export_the_same_span(self) -> None: + attributes: Final = mapped(_MODEL_CALL_WITH_CONTENT) + span_only: Final = INHERITING_DEST.model_copy( + update={"capture_message_content": CaptureMessageContent.SPAN_ONLY} + ) + omitted_exporter, span_only_exporter = InMemorySpanExporter(), InMemorySpanExporter() + + for destination, exporter in ((INHERITING_DEST, omitted_exporter), (span_only, span_only_exporter)): + self._run( + self._fan_out(InMemorySpanExporter(), {destination.endpoint: exporter}), (destination,), attributes + ) + + assert {n: dict(s.attributes) for n, s in by_name(omitted_exporter).items()} == { + n: dict(s.attributes) for n, s in by_name(span_only_exporter).items() + } + + @pytest.mark.parametrize( + ("globally_captured", "setting", "content_exported"), + [ + (True, "no_content", False), + (True, None, True), + (False, "span_only", True), + (False, None, False), + ], + ) + def test_the_operators_exporter_on_the_same_account_honors_the_teams_setting_once( + self, + monkeypatch: pytest.MonkeyPatch, + globally_captured: bool, + setting: str | None, + content_exported: bool, + ) -> None: + """The fan-out skips a destination the operator already writes to, so the operator's + copy is the one that account receives and must not bypass the team's restriction.""" + monkeypatch.setattr(litellm, "otel_tenant_destination_mode", "additive", raising=False) + shared: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor( + _OverriddenBackendFilter( + SimpleSpanProcessor(shared), + "langfuse_otel", + "full", + TestRoutingMode.OPERATOR_SINK, + global_captures=globally_captured, + ) + ) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(shared), + operator_sinks=MappingProxyType({TestRoutingMode.OPERATOR_SINK: "full"}), + ) + ) + destination: Final = OtelDestination( + endpoint=TestRoutingMode.SAME_ACCOUNT_ENDPOINT, + headers=MappingProxyType({"Authorization": "Basic op"}), + callback_name="langfuse_otel", + capture_message_content=setting, + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + + model_calls: Final = [s for s in shared.get_finished_spans() if s.name == "chat gpt-4o"] + assert len(model_calls) == 1, "the same account received the span twice" + assert carries_content(model_calls[0]) is content_exported + + @pytest.mark.parametrize("mode", ["override", "additive"]) + def test_an_operator_collector_on_the_teams_account_does_not_bypass_no_content( + self, monkeypatch: pytest.MonkeyPatch, mode: str + ) -> None: + monkeypatch.setattr(litellm, "otel_tenant_destination_mode", mode, raising=False) + shared: Final = InMemorySpanExporter() + kind: Final = f"lit8244_collector_{mode}" + register_exporter_factory(kind, lambda _spec: shared) + config: Final = OpenTelemetryV2Config( + exporters=[ + ExporterSpec( + kind=kind, + endpoint=TestRoutingMode.OPERATOR_SINK[0], + headers="authorization=Basic op", + ) + ] + ) + provider: Final = build_tracer_provider(config, use_simple_processor=True, tenant_overrides=True) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(shared), + operator_sinks=operator_sink_scopes(config), + ) + ) + destination: Final = OtelDestination( + endpoint=TestRoutingMode.SAME_ACCOUNT_ENDPOINT, + headers=MappingProxyType({"Authorization": "Basic op"}), + callback_name="langfuse_otel", + capture_message_content="no_content", + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + + model_calls: Final = [s for s in shared.get_finished_spans() if s.name == "chat gpt-4o"] + assert model_calls, "the account must receive the model call before its content can be judged absent" + assert not any(carries_content(span) for span in model_calls) + + @pytest.mark.parametrize( + ("global_capture", "content_exported"), + [(CaptureMessageContent.NO_CONTENT, False), (CaptureMessageContent.SPAN_ONLY, True)], + ) + def test_a_routed_provider_exports_content_only_when_the_global_setting_captures( + self, global_capture: str, content_exported: bool + ) -> None: + """A sibling destination's span_only makes the logger collect content, and a request + routed to this callback's team credentials must not carry it under a global no_content.""" + exporters: dict[ExporterOwner | None, InMemorySpanExporter] = {} + kind: Final = f"lit8244_routed_{global_capture}" + register_exporter_factory(kind, lambda spec: exporters.setdefault(spec.owner, InMemorySpanExporter())) + config: Final = OpenTelemetryV2Config( + capture_message_content=global_capture, + exporters=[ + ExporterSpec( + kind=kind, + endpoint="http://op.local", + owner=ExporterOwner.LANGFUSE_OTEL, + use_simple_processor=True, + ), + ExporterSpec(kind=kind), + ], + ) + cache: Final = TenantTracerCache(config, "langfuse_otel", "litellm") + + route: Final = cache.route_for( + get_tracer(TracerProvider(), "litellm"), + {"langfuse_public_key": "pk-team", "langfuse_secret_key": "sk-team"}, + ) + with route.tracer.start_as_current_span("chat gpt-4o") as llm: + llm.set_attributes(mapped(_MODEL_CALL_WITH_CONTENT)) + cache.release(route.provider) + + assert route.detached is True + for exporter in (exporters[ExporterOwner.LANGFUSE_OTEL], exporters[None]): + copy = by_name(exporter)["chat gpt-4o"] + assert carries_content(copy) is content_exported + assert copy.attributes["gen_ai.usage.input_tokens"] == 12 + + @pytest.mark.parametrize("first", ["newrelic", "langfuse_otel"]) + def test_an_omitted_setting_follows_its_own_callbacks_mode_whatever_the_callback_order( + self, monkeypatch: pytest.MonkeyPatch, first: str + ) -> None: + monkeypatch.delenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", raising=False) + tenant: Final = InMemorySpanExporter() + monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, "otlp_http", lambda _spec: tenant) + langfuse: Final = OpenTelemetryV2Config( + capture_message_content=CaptureMessageContent.SPAN_ONLY, + exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)], + ) + newrelic: Final = OpenTelemetryV2Config( + capture_message_content=CaptureMessageContent.NO_CONTENT, + exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.NEWRELIC)], + ) + provider: Final = TracerProvider() + attach_tenant_fan_out(provider, *((newrelic, langfuse) if first == "newrelic" else (langfuse, newrelic))) + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + provider.force_flush() + + assert carries_content(by_name(tenant)["chat gpt-4o"]) + + def test_an_omitted_setting_follows_the_capture_mode_set_in_callback_settings_otel( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", raising=False) + tenant: Final = InMemorySpanExporter() + monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, "otlp_http", lambda _spec: tenant) + operator: Final = OpenTelemetryV2Config(**{"capture_message_content": "span_only", "exporter": "in_memory"}) + provider: Final = TracerProvider() + attach_tenant_fan_out(provider, operator) + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + provider.force_flush() + + assert carries_content(by_name(tenant)["chat gpt-4o"]), "the YAML value is the global default, not only the env" + + def test_a_published_presets_capture_mode_is_the_default_for_an_unowned_destination( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A destination for a backend the proxy does not export to itself falls back to the published logger's mode, + here New Relic with record_content on, so it keeps the content.""" + monkeypatch.delenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", raising=False) + tenant: Final = InMemorySpanExporter() + monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, "otlp_http", lambda _spec: tenant) + newrelic: Final = OpenTelemetryV2Config( + capture_message_content=CaptureMessageContent.SPAN_ONLY, + exporters=[ + ExporterSpec(kind="console"), + ExporterSpec(kind="in_memory", owner=ExporterOwner.NEWRELIC), + ], + ) + provider: Final = TracerProvider() + attach_tenant_fan_out(provider, newrelic) + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + provider.force_flush() + + assert carries_content(by_name(tenant)["chat gpt-4o"]) + + @pytest.mark.usefixtures("allow_test_hosts") + @pytest.mark.parametrize("setting", ["no_content", "span_only", None]) + def test_the_setting_rides_the_teams_destination_and_omission_stays_omitted( + self, monkeypatch: pytest.MonkeyPatch, setting: str | None + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + callback_vars: Final = { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + } + if setting is not None: + callback_vars["capture_message_content"] = setting + auth: Final = UserAPIKeyAuth( + team_metadata={ + "logging": [ + {"callback_name": "langfuse_otel", "callback_type": "success", "callback_vars": callback_vars}, + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"arize_api_key": "k", "arize_space_id": "s"}, + }, + ] + } + ) + + destinations: Final = {d.callback_name: d for d in resolve_tenant_otel_destinations(auth)} + + assert destinations["langfuse_otel"].capture_message_content == setting + assert destinations["arize"].capture_message_content is None, "the setting stays on its own callback" + + @pytest.mark.usefixtures("allow_test_hosts") + @pytest.mark.parametrize("setting", ["event_only", "span_and_event"]) + def test_a_stored_event_mode_is_skipped_like_any_invalid_entry( + self, monkeypatch: pytest.MonkeyPatch, setting: str + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + auth: Final = UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + "capture_message_content": setting, + }, + }, + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"arize_api_key": "k", "arize_space_id": "s"}, + }, + ] + } + ) + + destinations: Final = [d.callback_name for d in resolve_tenant_otel_destinations(auth)] + + assert destinations == ["arize"] + + @pytest.mark.parametrize("value", ["full", "NO_CONTENT", "", "true", "event_only", "span_and_event"]) + def test_an_unsupported_value_fails_registration(self, value: str) -> None: + with pytest.raises( + ValueError, + match=r"Invalid capture_message_content .*\['no_content', 'span_only'\]", + ): + AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={ + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "capture_message_content": value, + }, + ) + + @pytest.mark.parametrize("value", ["no_content", "span_only"]) + def test_a_supported_value_is_stored_as_given(self, value: str) -> None: + saved: Final = AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "capture_message_content": value}, + ) + + assert saved.callback_vars["capture_message_content"] == value + + def test_a_team_wide_callback_vars_map_cannot_carry_the_setting(self) -> None: + """The flattened map is shared by every callback, so a value there would apply to all of them.""" + with pytest.raises(ValueError, match="Invalid callback variable: capture_message_content"): + TeamCallbackMetadata( + success_callback=["langfuse_otel", "signoz"], callback_vars={"capture_message_content": "span_only"} + ) + + def test_flattening_the_entries_leaves_the_setting_on_its_own_entry(self) -> None: + flattened: Final = convert_key_logging_metadata_to_callback( + AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={"langfuse_public_key": "pk", "capture_message_content": "span_only"}, + ), + None, + ) + + assert flattened.callback_vars == {"langfuse_public_key": "pk"} + + class TestArizeProjectRouting: """Arize rejects an export that names no project (400: set x-project-name, arize.project.name, openinference.project.name, or model_id), so an Arize diff --git a/tests/unit/integrations/otel/test_otel_v2_logger.py b/tests/unit/integrations/otel/test_otel_v2_logger.py index 3b1dd592d08..6a9a963948e 100644 --- a/tests/unit/integrations/otel/test_otel_v2_logger.py +++ b/tests/unit/integrations/otel/test_otel_v2_logger.py @@ -9,11 +9,13 @@ hooks, proxy SERVER span lifecycle (start + setters), parent-context resolution import ast import asyncio import contextlib +import contextvars import os import re from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path +from types import MappingProxyType from typing import Final from unittest.mock import patch @@ -43,7 +45,8 @@ from litellm.integrations.otel import ( # noqa: E402 OpenTelemetryV2Config, ) from litellm.integrations.otel.logger import OpenTelemetryV2 # noqa: E402 -from litellm.integrations.otel.model.config import ExporterSpec # noqa: E402 +from litellm.integrations.otel.model.config import CaptureMessageContent, ExporterSpec # noqa: E402 +from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402 from litellm.integrations.otel.model.spans import ( # noqa: E402 LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole, @@ -55,6 +58,7 @@ from litellm.integrations.otel.plumbing.context import ( # noqa: E402 reset_mcp_message_transport_span, set_mcp_message_trace_carrier, set_mcp_message_transport_span, + set_request_destinations, set_request_root_span, ) from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN # noqa: E402 @@ -617,8 +621,6 @@ def _mcp_payload(**overrides): def _logger_capturing(): - from litellm.integrations.otel.model.config import CaptureMessageContent - cfg = OpenTelemetryV2Config( exporter="in_memory", legacy_compat=False, @@ -684,6 +686,57 @@ def test_mcp_tool_call_captures_io_when_enabled(): assert "21" in span.attributes["gen_ai.tool.call.result"] +@pytest.mark.parametrize( + ("capture_mode", "captures_content"), + [ + (None, False), + (CaptureMessageContent.NO_CONTENT, False), + (CaptureMessageContent.SPAN_ONLY, True), + (CaptureMessageContent.EVENT_ONLY, False), + (CaptureMessageContent.SPAN_AND_EVENT, True), + ], +) +def test_only_a_request_destination_can_turn_on_llm_payload_content( + capture_mode: CaptureMessageContent | None, captures_content: bool +) -> None: + config: Final = OpenTelemetryV2Config( + exporter="in_memory", + capture_message_content=CaptureMessageContent.NO_CONTENT, + ) + exporter: Final = InMemorySpanExporter() + logger: Final = OpenTelemetryV2( + config=config, tracer_provider=providers.build_tracer_provider(config, exporter=exporter) + ) + payload: Final = _payload( + messages=[{"role": "user", "content": "prompt marker"}], + response={ + "id": "resp_1", + "model": "gpt-4o-2024", + "choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "answer marker"}}], + }, + ) + kwargs: Final = { + **_kwargs(payload), + "standard_callback_dynamic_params": {"capture_message_content": "span_only"}, + } + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + capture_message_content=capture_mode, + ) + + def emit() -> None: + set_request_destinations((destination,)) + _emit_llm(logger, kwargs) + + contextvars.copy_context().run(emit) + + (span,) = exporter.get_finished_spans() + assert ("gen_ai.input.messages" in span.attributes) is captures_content + assert ("gen_ai.output.messages" in span.attributes) is captures_content + + def test_mcp_tool_call_failure_marks_error(): logger, exporter = _logger() payload = _mcp_payload( diff --git a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py index ea0416cecbd..318859cb3ed 100644 --- a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py @@ -1692,6 +1692,19 @@ def test_capture_message_content_normalizer_only_touches_strings(): OpenTelemetryV2Config(capture_message_content=123) +def test_unknown_capture_message_content_env_is_kept_as_a_plain_string_that_captures_nothing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "Invalid-Mode") + + config: Final = OpenTelemetryV2Config() + + assert type(config.capture_message_content) is str + assert config.capture_message_content == "invalid-mode" + assert config.capture_span_content is False + assert config.model_dump()["capture_message_content"] == "invalid-mode" + + def test_v2_flag_is_off_by_default(monkeypatch): monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) is_otel_v2_enabled.cache_clear() diff --git a/tests/unit/llms/bedrock/passthrough/guardrail_translation/test_handler.py b/tests/unit/llms/bedrock/passthrough/guardrail_translation/test_handler.py index da7ed635dcb..57768b41943 100644 --- a/tests/unit/llms/bedrock/passthrough/guardrail_translation/test_handler.py +++ b/tests/unit/llms/bedrock/passthrough/guardrail_translation/test_handler.py @@ -10,9 +10,21 @@ Validates that: """ import copy -import pytest +from datetime import datetime +from typing import Literal from unittest.mock import AsyncMock, MagicMock +import pytest + +import litellm +from litellm.caching.caching import DualCache +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.utils import ProxyLogging +from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.utils import GenericGuardrailAPIInputs + from litellm.llms.bedrock.passthrough.guardrail_translation.handler import ( BedrockPassthroughGuardrailHandler, _extract_converse_texts, @@ -820,6 +832,52 @@ def _build_event_stream_frame(event_type: str, payload: dict) -> bytes: return prelude + prelude_crc_b + headers_bytes + payload_bytes + msg_crc_b +_TEST_CARD = "4111 1111 1111 1111" + + +class _CardMaskingGuardrail(CustomGuardrail): + def __init__(self) -> None: + super().__init__(guardrail_name="card-mask", event_hook=GuardrailEventHooks.post_call, default_on=True) + self.received_texts: list[list[str]] = [] + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + texts = inputs.get("texts", []) + self.received_texts.append(list(texts)) + return {**inputs, "texts": [text.replace(_TEST_CARD, "****") for text in texts]} + + +async def _run_post_call_guardrail_on_stream( + stream_bytes: bytes, guardrail: CustomGuardrail, monkeypatch: pytest.MonkeyPatch +) -> bytes: + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + logging_obj = LiteLLMLoggingObj( + model="claude", + messages=[], + stream=True, + call_type="allm_passthrough_route", + start_time=datetime(2026, 1, 1), + litellm_call_id="test-call", + function_id="test-function", + ) + return await BedrockPassthroughGuardrailHandler.de_anonymize_event_stream( + body_bytes=stream_bytes, + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + data={ + "model": "claude", + "custom_llm_provider": "bedrock", + "endpoint": "model/claude/converse-stream", + "litellm_logging_obj": logging_obj, + }, + ) + + class TestDeAnonymizeConverseStream: def _make_proxy_logging(self, mock_hook) -> MagicMock: proxy_logging_obj = MagicMock() @@ -870,6 +928,78 @@ class TestDeAnonymizeConverseStream: ] assert "".join(texts) == "John Doe works at Acme Corp" + @staticmethod + def _deltas_by_block(result: bytes, field: str) -> dict[int, str]: + import json + from botocore.eventstream import EventStreamBuffer + + buf = EventStreamBuffer() + buf.add_data(result) + blocks: dict[int, str] = {} + for msg in buf: + if msg.headers.get(":event-type") != "contentBlockDelta": + continue + payload = json.loads(msg.payload) + delta = payload["delta"] + value = delta["toolUse"]["input"] if field == "toolUse" else delta[field] + blocks[payload["contentBlockIndex"]] = blocks.get(payload["contentBlockIndex"], "") + value + return blocks + + @pytest.mark.asyncio + async def test_text_blocks_are_guardrailed_separately(self, monkeypatch): + stream_bytes = ( + _build_event_stream_frame("messageStart", {"role": "assistant"}) + + _build_event_stream_frame( + "contentBlockDelta", {"contentBlockIndex": 0, "delta": {"text": f"Card: {_TEST_CARD}."}} + ) + + _build_event_stream_frame("contentBlockStop", {"contentBlockIndex": 0}) + + _build_event_stream_frame( + "contentBlockDelta", {"contentBlockIndex": 1, "delta": {"text": "Thanks for sailing with us!"}} + ) + + _build_event_stream_frame("contentBlockStop", {"contentBlockIndex": 1}) + + _build_event_stream_frame("messageStop", {"stopReason": "end_turn"}) + ) + guardrail = _CardMaskingGuardrail() + + result = await _run_post_call_guardrail_on_stream(stream_bytes, guardrail, monkeypatch) + + assert guardrail.received_texts == [[f"Card: {_TEST_CARD}.", "Thanks for sailing with us!"]] + assert self._deltas_by_block(result, "text") == {0: "Card: ****.", 1: "Thanks for sailing with us!"} + + @pytest.mark.asyncio + async def test_parallel_tool_use_inputs_stay_in_their_own_block(self, monkeypatch): + import json + + first_input = f'{{"ship":"AU","cardNumber":"{_TEST_CARD}"}}' + second_input = '{"ship":"AU"}' + stream_bytes = ( + _build_event_stream_frame( + "contentBlockStart", + {"contentBlockIndex": 0, "start": {"toolUse": {"toolUseId": "tooluse_a", "name": "book"}}}, + ) + + _build_event_stream_frame( + "contentBlockDelta", {"contentBlockIndex": 0, "delta": {"toolUse": {"input": first_input}}} + ) + + _build_event_stream_frame("contentBlockStop", {"contentBlockIndex": 0}) + + _build_event_stream_frame( + "contentBlockStart", + {"contentBlockIndex": 1, "start": {"toolUse": {"toolUseId": "tooluse_b", "name": "ship_info"}}}, + ) + + _build_event_stream_frame( + "contentBlockDelta", {"contentBlockIndex": 1, "delta": {"toolUse": {"input": second_input}}} + ) + + _build_event_stream_frame("contentBlockStop", {"contentBlockIndex": 1}) + + _build_event_stream_frame("messageStop", {"stopReason": "tool_use"}) + ) + guardrail = _CardMaskingGuardrail() + + result = await _run_post_call_guardrail_on_stream(stream_bytes, guardrail, monkeypatch) + + assert guardrail.received_texts == [[first_input, second_input]] + inputs = self._deltas_by_block(result, "toolUse") + assert json.loads(inputs[0]) == {"ship": "AU", "cardNumber": "****"} + assert json.loads(inputs[1]) == {"ship": "AU"} + @pytest.mark.asyncio async def test_tokens_split_across_chunks_reassembled(self): import json diff --git a/tests/unit/llms/scaledown/__init__.py b/tests/unit/llms/scaledown/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/scaledown/chat/__init__.py b/tests/unit/llms/scaledown/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/scaledown/chat/test_scaledown_chat_transformation.py b/tests/unit/llms/scaledown/chat/test_scaledown_chat_transformation.py new file mode 100644 index 00000000000..82f03a2cd80 --- /dev/null +++ b/tests/unit/llms/scaledown/chat/test_scaledown_chat_transformation.py @@ -0,0 +1,1213 @@ +import json +from collections.abc import Mapping +from queue import SimpleQueue +from typing import Final, Literal + +import httpx +import pytest +import respx +from pydantic import JsonValue + +import litellm +from litellm.llms.scaledown.chat.transformation import ( + DECISIONS_UPSTREAM_MODEL, + ScaleDownChatConfig, + ScaleDownError, +) +from litellm.types.utils import ModelResponse + +BASE = "https://api.scaledown.xyz" + + +@pytest.fixture +def config() -> ScaleDownChatConfig: + return ScaleDownChatConfig() + + +@pytest.fixture(autouse=True) +def scaledown_api_key(monkeypatch): + monkeypatch.setenv("SCALEDOWN_API_KEY", "sk-scaledown-test") + monkeypatch.delenv("SCALEDOWN_API_BASE", raising=False) + + +def _transform_response( + config: ScaleDownChatConfig, model: str, payload: dict, request_data: dict | None = None +) -> ModelResponse: + return config.transform_response( + model=model, + raw_response=httpx.Response( + status_code=200, + json=payload, + request=httpx.Request("POST", f"{BASE}/extract"), + ), + model_response=ModelResponse(), + logging_obj=None, + request_data=request_data or {}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + +def _build_decisions(config: ScaleDownChatConfig, messages: list, optional_params: dict) -> dict: + body = config.transform_request( + model="scaledown/decisions", messages=messages, optional_params=optional_params, litellm_params={}, headers={} + ) + config.sign_request( + headers={}, + optional_params=optional_params, + request_data=body, + api_base=f"{BASE}/v1/scaledown", + model="scaledown/decisions", + ) + return body + + +def test_auth_uses_x_api_key_not_bearer(config): + headers = config.validate_environment( + headers={}, + model="scaledown/extract", + messages=[], + optional_params={}, + litellm_params={}, + api_key="sk-explicit", + ) + + assert headers["x-api-key"] == "sk-explicit" + assert "Authorization" not in headers + + +def test_missing_api_key_is_rejected_before_any_request(config, monkeypatch): + monkeypatch.delenv("SCALEDOWN_API_KEY", raising=False) + + with pytest.raises(ScaleDownError) as exc: + config.validate_environment( + headers={}, + model="scaledown/extract", + messages=[], + optional_params={}, + litellm_params={}, + ) + + assert exc.value.status_code == 401 + + +@pytest.mark.parametrize( + "model, path", + [ + ("scaledown/extract", "/extract"), + ("scaledown/summarize", "/summarization/abstractive"), + ("scaledown/compress", "/compress/raw/"), + ("scaledown/classify", "/classify"), + ("scaledown/decisions", "/v1/scaledown"), + ], +) +def test_each_model_targets_its_native_endpoint(config, model, path): + url = config.get_complete_url(api_base=None, api_key="k", model=model, optional_params={}, litellm_params={}) + + assert url == f"{BASE}{path}" + + +@pytest.mark.parametrize( + "api_base", + ["https://staging.scaledown.xyz", "https://staging.scaledown.xyz/", "https://staging.scaledown.xyz/v1"], +) +def test_api_base_with_or_without_v1_resolves_to_the_same_host(config, api_base): + extract = config.get_complete_url( + api_base=api_base, api_key="k", model="scaledown/extract", optional_params={}, litellm_params={} + ) + decisions = config.get_complete_url( + api_base=api_base, api_key="k", model="scaledown/decisions", optional_params={}, litellm_params={} + ) + + assert extract == "https://staging.scaledown.xyz/extract" + assert decisions == "https://staging.scaledown.xyz/v1/scaledown" + + +def test_decisions_request_carries_state_and_questions_without_chat_keys(config): + questions = { + "category": { + "type": "choice", + "instructions": "Which category?", + "criteria": {"billing": "A charge or refund.", "technical": "A bug."}, + } + } + + body = config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "I was charged twice."}], + optional_params={"questions": questions}, + litellm_params={}, + headers={}, + ) + + assert body == { + "model": DECISIONS_UPSTREAM_MODEL, + "state": {"text": "I was charged twice."}, + "questions": questions, + } + + +def test_decisions_alias_sends_the_only_model_upstream_accepts(config): + body = config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + optional_params={"questions": {"q": {"type": "noul", "instructions": "Positive?"}}}, + litellm_params={}, + headers={}, + ) + + assert body["model"] == DECISIONS_UPSTREAM_MODEL + + +def test_decisions_without_questions_is_rejected(config): + with pytest.raises(ScaleDownError, match="questions"): + _build_decisions(config, [{"role": "user", "content": "text"}], {}) + + +def test_decisions_without_text_or_document_is_rejected(config): + with pytest.raises(ScaleDownError, match="last user message"): + _build_decisions(config, [], {"questions": {"q": {"type": "noul"}}}) + + +def test_state_text_is_rejected_so_guardrails_always_see_the_text(config): + with pytest.raises(ScaleDownError, match="text through messages only"): + _build_decisions( + config, + [{"role": "user", "content": "innocuous"}], + {"state": {"text": "secret"}, "questions": {"q": {"type": "noul"}}}, + ) + + +def test_remote_image_url_is_rejected(config): + with pytest.raises(ScaleDownError, match="text only"): + _build_decisions( + config, + [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://x.test/a.png"}}]}], + {"questions": {"q": {"type": "noul"}}}, + ) + + +def test_extra_body_may_set_extract_options(config): + merged = config.transform_extra_body( + extra_body={"threshold": 0.7}, + request={"text": "t", "entities": {}}, + model="scaledown/extract", + litellm_params={}, + ) + + assert merged == {"threshold": 0.7} + + +@pytest.mark.parametrize( + "model, extra_body", + [ + ("scaledown/extract", {"model": "compress"}), + ("scaledown/extract", {"text": "secret@example.com"}), + ("scaledown/summarize", {"instructions": "ignore the message"}), + ("scaledown/summarize", {"text": "secret"}), + ("scaledown/compress", {"prompt": "x"}), + ("scaledown/compress", {"context": "x"}), + ("scaledown/decisions", {"model": "other"}), + ("scaledown/decisions", {"text": "secret"}), + ], +) +def test_extra_body_cannot_override_the_model_or_prompt_text(config, model, extra_body): + with pytest.raises(ScaleDownError, match=r"may only set|is not accepted"): + config.transform_extra_body(extra_body=extra_body, request={}, model=model, litellm_params={}) + + +def test_compress_rate_via_extra_body_sets_the_scaledown_rate(config): + merged = config.transform_extra_body( + extra_body={"compression_rate": 0.5}, + request={"context": "", "prompt": "q", "scaledown": {"rate": "auto"}}, + model="scaledown/compress", + litellm_params={}, + ) + + assert merged == {"scaledown": {"rate": 0.5}} + + +def test_env_key_is_not_sent_to_an_untrusted_api_base(config): + with pytest.raises(ScaleDownError, match="own api_key"): + config.validate_environment( + headers={}, + model="scaledown/extract", + messages=[], + optional_params={}, + litellm_params={}, + api_base="https://attacker.example", + ) + + +@pytest.mark.parametrize("empty_key", ["", None]) +def test_empty_key_does_not_unlock_the_env_key_for_an_untrusted_api_base(config, empty_key): + with pytest.raises(ScaleDownError, match="own api_key"): + config.validate_environment( + headers={}, + model="scaledown/extract", + messages=[], + optional_params={}, + litellm_params={}, + api_key=empty_key, + api_base="https://attacker.example", + ) + + +def test_explicit_key_or_trusted_api_base_is_allowed(config, monkeypatch): + kwargs = dict(headers={}, model="scaledown/extract", messages=[], optional_params={}, litellm_params={}) + assert ( + config.validate_environment(api_key="own", api_base="https://elsewhere.example", **kwargs)["x-api-key"] == "own" + ) + assert config.validate_environment(api_base=f"{BASE}/v1", **kwargs)["x-api-key"] == "sk-scaledown-test" + monkeypatch.setenv("SCALEDOWN_API_BASE", "https://staging.scaledown.xyz") + assert ( + config.validate_environment(api_base="https://staging.scaledown.xyz/", **kwargs)["x-api-key"] + == "sk-scaledown-test" + ) + + +def test_max_completion_tokens_maps_to_max_tokens(config): + assert config.map_openai_params( + non_default_params={"max_completion_tokens": 50}, + optional_params={}, + model="scaledown/summarize", + drop_params=False, + ) == {"max_tokens": 50} + + +def test_router_params_are_accepted_and_not_forwarded(config): + assert ( + config.map_openai_params( + non_default_params={"max_retries": 2}, optional_params={}, model="scaledown/decisions", drop_params=False + ) + == {} + ) + + +def test_every_model_fakes_a_stream(config): + assert config.should_fake_stream(model="scaledown/compress", stream=True) + assert not config.should_fake_stream(model="scaledown/compress", stream=False) + + +def test_unknown_question_type_is_rejected(config): + with pytest.raises(ScaleDownError, match="expected one of"): + config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "t"}], + optional_params={"questions": {"q": {"type": "ranking"}}}, + litellm_params={}, + headers={}, + ) + + +def test_choice_question_without_criteria_is_rejected(config): + with pytest.raises(ScaleDownError, match="non-empty 'criteria' map"): + config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "t"}], + optional_params={"questions": {"q": {"type": "choice", "criteria": {}}}}, + litellm_params={}, + headers={}, + ) + + +@pytest.mark.parametrize("levels", [1, 11]) +def test_score_criteria_outside_two_to_ten_is_rejected(config, levels): + with pytest.raises(ScaleDownError, match="ordered list"): + config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "t"}], + optional_params={"questions": {"q": {"type": "score", "criteria": ["l"] * levels}}}, + litellm_params={}, + headers={}, + ) + + +@pytest.mark.parametrize("levels", [2, 10]) +def test_score_criteria_within_two_to_ten_is_accepted(config, levels): + criteria = [f"level {index}" for index in range(levels)] + + body = config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "t"}], + optional_params={"questions": {"q": {"type": "score", "criteria": criteria}}}, + litellm_params={}, + headers={}, + ) + + assert body["questions"]["q"]["criteria"] == criteria + + +def test_extract_content_is_the_clean_fields_and_raw_payload_is_kept(config): + payload = { + "entities": [], + "structured_result": { + "invoice": { + "vendor": "Northwind", + "vendor_span_anchor": "Invoice from Northwind", + "customer": {"_value": "Ada Lovelace", "_span_anchor": "to Ada Lovelace"}, + "amount": 500, + } + }, + "input_tokens": 171, + } + + requested = {"invoice": {"vendor": "v", "customer": "c", "amount": "a"}} + response = _transform_response(config, "scaledown/extract", payload, {"entities": requested}) + + assert json.loads(response.choices[0].message.content) == { + "invoice": {"vendor": "Northwind", "customer": "Ada Lovelace", "amount": 500} + } + assert response.model == "scaledown/extract" + assert response.usage.prompt_tokens == 171 + assert response.usage.completion_tokens == 0 + assert response._hidden_params["scaledown_response"] == payload + + +def test_extract_cleaning_keeps_requested_fields_named_like_provider_keys(config): + payload = {"structured_result": {"_value": "x", "vendor": "y", "note_span_anchor": "z", "note": "n"}} + requested = {"_value": "v", "vendor": "v", "note_span_anchor": "s", "note": "n"} + + response = _transform_response(config, "scaledown/extract", payload, {"entities": requested}) + + assert json.loads(response.choices[0].message.content) == payload["structured_result"] + + +def test_extract_cleaning_handles_lists_of_objects(config): + payload = {"structured_result": {"items": [{"sku": "a", "sku_span_anchor": "s"}, {"sku": {"_value": "b"}}]}} + + response = _transform_response(config, "scaledown/extract", payload, {"entities": {"items": [{"sku": "code"}]}}) + + assert json.loads(response.choices[0].message.content) == {"items": [{"sku": "a"}, {"sku": "b"}]} + + +def test_long_ref_chain_is_rejected_cleanly_not_with_a_recursion_error(config): + defs = {f"D{i}": {"$ref": f"#/$defs/D{i + 1}"} for i in range(5000)} + defs["D5000"] = {"type": "string"} + schema = {"type": "object", "$defs": defs, "properties": {"f": {"$ref": "#/$defs/D0"}}} + + with pytest.raises(ScaleDownError, match="chained"): + _extract_body(config, schema) + + +def test_deeply_nested_schema_is_rejected_cleanly(config): + nested: dict = {"type": "string"} + for _ in range(200): + nested = {"type": "object", "properties": {"x": nested}} + schema = {"type": "object", "properties": {"root": nested}} + + with pytest.raises(ScaleDownError, match="nests deeper"): + _extract_body(config, schema) + + +def test_more_than_one_image_is_rejected(config): + image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGVsbG8="}} + + with pytest.raises(ScaleDownError, match="text only"): + _build_decisions(config, [{"role": "user", "content": [image, image]}], {"questions": {"q": {"type": "noul"}}}) + + +def test_stream_options_is_accepted_and_not_forwarded(config): + assert ( + config.map_openai_params( + non_default_params={"stream_options": {"include_usage": True}}, + optional_params={}, + model="scaledown/summarize", + drop_params=False, + ) + == {} + ) + + +@pytest.mark.parametrize("wrapper", [None, "anyOf", "oneOf"]) +def test_extract_schema_follows_local_refs(config: ScaleDownChatConfig, wrapper: str | None) -> None: + address: Final = {"$ref": "#/$defs/Address"} + schema: Final = { + "type": "object", + "$defs": { + "Address": {"type": "object", "properties": {"city": {"type": "string", "description": "city name"}}} + }, + "properties": { + "address": {wrapper: [address, {"type": "null"}]} if wrapper else address, + "name": {"type": "string"}, + }, + } + + body = config.transform_request( + model="scaledown/extract", + messages=[{"role": "user", "content": "text"}], + optional_params={"response_format": {"type": "json_schema", "json_schema": {"name": "n", "schema": schema}}}, + litellm_params={}, + headers={}, + ) + + assert body["entities"] == {"address": {"city": "city name"}, "name": "name"} + + +@pytest.mark.parametrize( + ("definition", "reference"), + [ + ("Invoice/Details", "#/$defs/Invoice~1Details"), + ("Invoice~Details", "#/$defs/Invoice~0Details"), + ("Invoice~1 Details", "#/%24defs/Invoice~01%20Details"), + ], +) +def test_extract_preserves_fields_from_escaped_refs( + config: ScaleDownChatConfig, definition: str, reference: str +) -> None: + schema: Final = { + "type": "object", + "$defs": {definition: {"type": "object", "properties": {"amount": {"type": "number"}}}}, + "properties": {"invoice": {"$ref": reference}}, + } + + assert _extract_body(config, schema)["entities"] == {"invoice": {"amount": "amount"}} + + +@pytest.mark.parametrize("reference", ["#/$defs/Missing", "https://example.com/schema", "#named-anchor", None, []]) +def test_extract_rejects_unresolvable_refs(config: ScaleDownChatConfig, reference: JsonValue) -> None: + schema: Final = {"type": "object", "properties": {"invoice": {"$ref": reference}}} + + with pytest.raises(ScaleDownError, match=r"\$ref") as exc: + _extract_body(config, schema) + + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize("field_type", ["array", ["array", "null"]]) +def test_extract_nullable_array_keeps_item_fields(config: ScaleDownChatConfig, field_type: str | list[str]) -> None: + schema: Final = { + "type": "object", + "properties": { + "invoices": { + "type": field_type, + "items": {"type": "object", "properties": {"amount": {"type": "number"}}}, + } + }, + } + + assert _extract_body(config, schema)["entities"] == {"invoices": [{"amount": "amount"}]} + + +@pytest.mark.parametrize("wrapper", ["anyOf", "oneOf", "$ref"]) +def test_extract_rejects_sibling_fields_instead_of_dropping_them(config: ScaleDownChatConfig, wrapper: str) -> None: + address: Final = {"type": "object", "properties": {"city": {"type": "string"}}} + reference: Final = {"$ref": "#/$defs/Address"} + field: Final = { + **(reference if wrapper == "$ref" else {wrapper: [address, {"type": "null"}]}), + "properties": {"postal_code": {"type": "string"}}, + } + schema: Final = {"$defs": {"Address": address}, "type": "object", "properties": {"address": field}} + + with pytest.raises(ScaleDownError, match="sibling") as exc: + _extract_body(config, schema) + + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize( + "field", + [ + {"anyOf": [{"type": "object", "properties": {"x": {"type": "string"}}}, {"type": "string"}]}, + {"oneOf": [{"type": "string"}]}, + {"anyOf": [False, {"type": "string"}]}, + {"allOf": [{"type": "object", "properties": {"x": {"type": "string"}}}]}, + ], +) +def test_extract_rejects_unsupported_composition(config: ScaleDownChatConfig, field: dict[str, JsonValue]) -> None: + with pytest.raises(ScaleDownError) as exc: + _extract_body(config, {"type": "object", "properties": {"field": field}}) + + assert exc.value.status_code == 400 + + +@respx.mock +@pytest.mark.parametrize("question_type", [[], {}]) +def test_malformed_question_types_are_request_errors(question_type: list[JsonValue] | dict[str, JsonValue]) -> None: + with pytest.raises(litellm.BadRequestError, match="Question 'q'"): + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": question_type}}, + ) + + assert not respx.calls + + +@respx.mock +def test_unknown_scaledown_model_is_a_request_error() -> None: + with pytest.raises(litellm.BadRequestError, match="Unknown ScaleDown model"): + litellm.completion(model="scaledown/unknown", messages=[{"role": "user", "content": "text"}]) + + assert not respx.calls + + +def _extract_body(config: ScaleDownChatConfig, schema: dict) -> dict: + return config.transform_request( + model="scaledown/extract", + messages=[{"role": "user", "content": "text"}], + optional_params={"response_format": {"type": "json_schema", "json_schema": {"name": "n", "schema": schema}}}, + litellm_params={}, + headers={}, + ) + + +def test_recursive_schema_is_cut_at_the_cycle_instead_of_expanded(config): + children = {name: {"$ref": "#/$defs/Node"} for name in ("a", "b", "c", "d")} + schema = { + "type": "object", + "$defs": {"Node": {"type": "object", "properties": children}}, + "properties": {"root": {"$ref": "#/$defs/Node"}}, + } + + entities = _extract_body(config, schema)["entities"] + + assert entities == {"root": {"a": "a", "b": "b", "c": "c", "d": "d"}} + + +def test_heavily_repeated_acyclic_refs_are_rejected_by_the_entity_budget(config): + defs = { + f"L{level}": {"type": "object", "properties": {name: {"$ref": f"#/$defs/L{level + 1}"} for name in "abcd"}} + for level in range(10) + } + defs["L10"] = {"type": "string", "description": "leaf"} + schema = {"type": "object", "$defs": defs, "properties": {"root": {"$ref": "#/$defs/L0"}}} + + with pytest.raises(ScaleDownError, match="expands to more than"): + _extract_body(config, schema) + + +def test_summarize_response_is_the_native_payload(config): + payload = {"summary": "terse summary", "input_chars": 400, "output_chars": 13, "input_tokens": 78} + + response = _transform_response(config, "scaledown/summarize", payload) + + assert json.loads(response.choices[0].message.content) == payload + assert response.usage.prompt_tokens == 78 + + +def test_compress_response_reports_top_level_input_tokens(config): + payload = { + "results": {"success": True, "compressed_prompt": "short", "original_prompt_tokens": 52}, + "input_tokens": 52, + } + + response = _transform_response(config, "scaledown/compress", payload) + + assert json.loads(response.choices[0].message.content) == payload + assert response.usage.prompt_tokens == 52 + + +def test_compress_response_falls_back_to_original_prompt_tokens(config): + payload = {"results": {"compressed_prompt": "short", "original_prompt_tokens": 900}} + + assert _transform_response(config, "scaledown/compress", payload).usage.prompt_tokens == 900 + + +def test_extract_request_maps_schema_to_entities_including_nesting(config): + schema = { + "type": "object", + "properties": { + "vendor": {"type": "string", "description": "company that issued the invoice"}, + "notes": {"type": "string"}, + "address": {"type": "object", "properties": {"city": {"type": "string", "description": "city name"}}}, + "line_items": { + "type": "array", + "items": {"type": "object", "properties": {"sku": {"type": "string", "description": "item code"}}}, + }, + }, + } + + body = config.transform_request( + model="scaledown/extract", + messages=[{"role": "user", "content": "Invoice from Northwind."}], + optional_params={"response_format": {"type": "json_schema", "json_schema": {"name": "i", "schema": schema}}}, + litellm_params={}, + headers={}, + ) + + assert body == { + "text": "Invoice from Northwind.", + "entities": { + "vendor": "company that issued the invoice", + "notes": "notes", + "address": {"city": "city name"}, + "line_items": [{"sku": "item code"}], + }, + } + + +def test_extract_without_a_schema_is_rejected(config): + with pytest.raises(ScaleDownError, match="response_format"): + config.transform_request( + model="scaledown/extract", + messages=[{"role": "user", "content": "text"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + +@pytest.mark.parametrize("role", ["system", "developer"]) +def test_summarize_request_splits_instructions_text_and_max_tokens( + config: ScaleDownChatConfig, role: Literal["system", "developer"] +) -> None: + body: Final = config.transform_request( + model="scaledown/summarize", + messages=[ + {"role": role, "content": "Be terse."}, + {"role": "user", "content": "a long document"}, + ], + optional_params={"max_tokens": 40}, + litellm_params={}, + headers={}, + ) + + assert body == {"text": "a long document", "instructions": "Be terse.", "max_tokens": 40} + + +def test_compress_request_puts_earlier_messages_in_context_and_defaults_rate_to_auto(config): + body = config.transform_request( + model="scaledown/compress", + messages=[ + {"role": "system", "content": "background"}, + {"role": "user", "content": "the question"}, + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert body == {"context": "background", "prompt": "the question", "scaledown": {"rate": "auto"}} + + +def test_decisions_response_ignores_the_upstream_cost_field(config): + payload = { + "model": "classify-1", + "answers": { + "category": { + "type": "choice", + "choice": "billing", + "probabilities": {"billing": 0.93, "technical": 0.07}, + "confidence": 0.93, + } + }, + "usage": {"input_tokens": 62, "output_tokens": 1, "cost": 0.00543}, + } + + response = _transform_response(config, "scaledown/decisions", payload) + + assert json.loads(response.choices[0].message.content) == payload["answers"] + assert response.usage.prompt_tokens == 62 + assert response.usage.completion_tokens == 1 + assert "response_cost" not in response._hidden_params + assert response._hidden_params["scaledown_response"] == payload + + +def test_score_answer_survives_the_envelope_intact(config): + answer = { + "type": "score", + "score": 1.43, + "confidence": 0.57, + "legend": {"0": "Cosmetic", "1": "Workaround", "2": "Blocking"}, + "probabilities": {"0": 0.0, "1": 0.57, "2": 0.43}, + } + payload = { + "model": "classify-1", + "answers": {"bug_severity": answer}, + "usage": {"input_tokens": 68, "output_tokens": 1, "cost": 0.0000029}, + } + + response = _transform_response(config, "scaledown/decisions", payload) + + assert json.loads(response.choices[0].message.content)["bug_severity"] == answer + + +def test_decisions_response_without_answers_is_an_error(config): + with pytest.raises(ScaleDownError, match="no answers"): + _transform_response(config, "scaledown/decisions", {"model": "classify-1"}) + + +def test_empty_native_response_is_an_error(config): + with pytest.raises(ScaleDownError, match="empty or malformed"): + _transform_response(config, "scaledown/extract", {}) + + +@pytest.mark.parametrize("upstream_status", [200, 502]) +def test_non_json_response_is_an_upstream_error(config: ScaleDownChatConfig, upstream_status: int) -> None: + raw_response: Final = httpx.Response( + status_code=upstream_status, + text="bad gateway", + request=httpx.Request("POST", f"{BASE}/v1/scaledown"), + ) + + with pytest.raises(ScaleDownError) as exc: + config.transform_response( + model="scaledown/decisions", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=None, + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert exc.value.status_code == 502 + + +def test_decisions_takes_no_openai_sampling_params(config): + assert config.get_supported_openai_params("scaledown/decisions") == ["stream"] + + +def test_unsupported_param_is_rejected_unless_dropped(config): + with pytest.raises(ScaleDownError, match="does not support"): + config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model="scaledown/decisions", + drop_params=False, + ) + + assert ( + config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model="scaledown/decisions", + drop_params=True, + ) + == {} + ) + + +@respx.mock +def test_completion_sends_x_api_key_to_the_native_summarize_endpoint(): + route = respx.post(f"{BASE}/summarization/abstractive").mock( + return_value=httpx.Response(200, json={"summary": "s", "input_tokens": 120}) + ) + + response = litellm.completion( + model="scaledown/summarize", + messages=[ + {"role": "system", "content": "Be terse."}, + {"role": "user", "content": "a long document"}, + ], + ) + + request = route.calls[0].request + assert request.headers["x-api-key"] == "sk-scaledown-test" + assert "authorization" not in request.headers + assert json.loads(request.content) == {"text": "a long document", "instructions": "Be terse."} + assert json.loads(response.choices[0].message.content)["summary"] == "s" + + +@respx.mock +def test_completion_routes_decisions_to_the_decisions_endpoint(): + route = respx.post(f"{BASE}/v1/scaledown").mock( + return_value=httpx.Response( + 200, + json={ + "model": "classify-1", + "answers": { + "category": { + "type": "choice", + "choice": "billing", + "probabilities": {"billing": 0.93, "technical": 0.07}, + "confidence": 0.93, + } + }, + "usage": {"input_tokens": 62, "output_tokens": 1, "cost": 0.00543}, + }, + ) + ) + + response = litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "I was charged twice."}], + questions={ + "category": { + "type": "choice", + "criteria": {"billing": "A charge or refund.", "technical": "A bug."}, + } + }, + ) + + sent = json.loads(route.calls[0].request.content) + assert sent["model"] == "classify-1" + assert sent["state"] == {"text": "I was charged twice."} + assert "messages" not in sent + assert json.loads(response.choices[0].message.content)["category"]["choice"] == "billing" + + +@respx.mock +def test_completion_extract_converts_response_format_to_entities(): + schema = { + "type": "json_schema", + "json_schema": { + "name": "invoice", + "schema": { + "type": "object", + "properties": {"vendor": {"type": "string", "description": "company name"}}, + }, + }, + } + extract_result = {"entities": [], "structured_result": {"vendor": "Acme Corp"}, "input_tokens": 20} + route = respx.post(f"{BASE}/extract").mock(return_value=httpx.Response(200, json=extract_result)) + + response = litellm.completion( + model="scaledown/extract", + messages=[{"role": "user", "content": "Acme Corp invoiced $500."}], + response_format=schema, + ) + + assert json.loads(route.calls[0].request.content) == { + "text": "Acme Corp invoiced $500.", + "entities": {"vendor": "company name"}, + } + assert json.loads(response.choices[0].message.content) == {"vendor": "Acme Corp"} + assert response._hidden_params["scaledown_response"] == extract_result + + +@respx.mock +def test_completion_cost_comes_from_input_tokens_not_upstream_usage_cost(): + respx.post(f"{BASE}/v1/scaledown").mock( + return_value=httpx.Response( + 200, + json={ + "model": "classify-1", + "answers": {"q": {"type": "noul", "noul": 0.8}}, + "usage": {"input_tokens": 1000, "output_tokens": 1, "cost": 99.0}, + }, + ) + ) + + response = litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": "noul", "instructions": "Positive?"}}, + ) + + cost = litellm.completion_cost(completion_response=response, model="scaledown/decisions") + assert cost == pytest.approx(1000 * litellm.model_cost["scaledown/decisions"]["input_cost_per_token"]) + + +@respx.mock +def test_completion_honors_scaledown_api_base(monkeypatch): + monkeypatch.setenv("SCALEDOWN_API_BASE", "https://staging.scaledown.xyz/v1") + route = respx.post("https://staging.scaledown.xyz/v1/scaledown").mock( + return_value=httpx.Response( + 200, + json={ + "model": "classify-1", + "answers": {"q": {"type": "noul", "noul": 0.8}}, + "usage": {"input_tokens": 10, "output_tokens": 1, "cost": 0.0}, + }, + ) + ) + + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": "noul", "instructions": "Positive?"}}, + ) + + assert route.called + + +@respx.mock +def test_upstream_payment_error_surfaces_the_detail(): + respx.post(f"{BASE}/v1/scaledown").mock(return_value=httpx.Response(402, json={"detail": "Insufficient credits"})) + + with pytest.raises(litellm.exceptions.BadRequestError) as exc: + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": "noul", "instructions": "Positive?"}}, + ) + + assert "Insufficient credits" in str(exc.value) + + +@respx.mock +def test_failed_completion_invokes_post_api_logger() -> None: + events: Final = SimpleQueue[str]() + + def logger_fn(details: Mapping[str, object]) -> None: + if details.get("log_event_type") == "post_api_call": + events.put(str(details.get("original_response"))) + + respx.post(f"{BASE}/v1/scaledown").respond(402, json={"detail": "Insufficient credits"}) + + with pytest.raises(litellm.BadRequestError): + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": "noul", "instructions": "Positive?"}}, + logger_fn=logger_fn, + ) + + assert not events.empty() + assert "Insufficient credits" in events.get_nowait() + + +@respx.mock +def test_malformed_decisions_request_never_reaches_the_network(): + route = respx.post(f"{BASE}/v1/scaledown") + + with pytest.raises(Exception, match="questions"): + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + ) + + assert not route.called + + +@respx.mock +def test_questions_passed_through_extra_body_reach_scaledown(): + route = respx.post(f"{BASE}/v1/scaledown").mock( + return_value=httpx.Response( + 200, + json={ + "model": "classify-1", + "answers": {"q": {"type": "noul", "noul": 0.8}}, + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + ) + ) + questions = {"q": {"type": "noul", "instructions": "Positive?"}} + + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + extra_body={"questions": questions}, + ) + + sent = json.loads(route.calls[0].request.content) + assert sent == {"model": DECISIONS_UPSTREAM_MODEL, "state": {"text": "text"}, "questions": questions} + + +@respx.mock +def test_streaming_request_is_served_as_a_single_chunk(): + respx.post(f"{BASE}/compress/raw/").mock( + return_value=httpx.Response(200, json={"compressed_prompt": "c", "original_prompt_tokens": 9}) + ) + + chunks = list( + litellm.completion( + model="scaledown/compress", + messages=[{"role": "user", "content": "the question"}], + stream=True, + ) + ) + + assert "compressed_prompt" in "".join(chunk.choices[0].delta.content or "" for chunk in chunks) + + +@pytest.mark.parametrize( + "structured,expected", + [ + (None, {"vendor": "Northwind"}), + ( + {"invoice": {"amount": 500, "amount_span_anchor": "$500"}}, + {"vendor": "Northwind", "invoice": {"amount": 500}}, + ), + ], +) +def test_extract_merges_first_scalar_match_with_nested_fields(config, structured, expected): + payload = { + "entities": [ + {"type": "vendor", "text": "Northwind", "confidence": 1.0}, + {"type": "vendor", "text": "Contoso", "confidence": 0.8}, + {"type": "unrequested", "text": "ignore"}, + ], + "structured_result": structured, + "input_tokens": 161, + } + response = _transform_response( + config, + "scaledown/extract", + payload, + {"entities": {"vendor": "company name", "missing": "absent field", "invoice": {"amount": "total"}}}, + ) + + assert json.loads(response.choices[0].message.content) == expected + assert response._hidden_params["scaledown_response"] == payload + assert response.usage.prompt_tokens == 161 + + +@respx.mock +def test_strict_extraction_is_rejected_before_network(): + route = respx.post(f"{BASE}/extract") + with pytest.raises(litellm.BadRequestError, match="strict JSON schemas"): + litellm.completion( + model="scaledown/extract", + messages=[{"role": "user", "content": "Invoice from Northwind. Total: $500."}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "invoice", + "strict": True, + "schema": {"type": "object", "properties": {"amount": {"type": "string"}}, "required": ["amount"]}, + }, + }, + ) + assert not route.called + + +@pytest.mark.parametrize("model", ["classify", "decisions", "extract", "summarize", "compress"]) +@pytest.mark.parametrize( + "content", + [ + [{"type": "image_url", "image_url": {"url": "data:image/png;base64,aGVsbG8="}}], + [ + {"type": "text", "text": "invoice"}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,aGVsbG8="}}, + ], + ], +) +def test_multimodal_inputs_are_rejected_instead_of_dropped(config, model, content): + with pytest.raises(ScaleDownError, match="text only"): + config.transform_request(f"scaledown/{model}", [{"role": "user", "content": content}], {}, {}, {}) + + +@pytest.mark.parametrize("params", [{"document": "QQ=="}, {"state": {"document": "QQ=="}}, {"text": "override"}]) +def test_native_input_overrides_are_rejected(config, params): + with pytest.raises(ScaleDownError, match="text through messages only"): + config.transform_request("scaledown/decisions", [{"role": "user", "content": "text"}], params, {}, {}) + + +@pytest.mark.parametrize("model", ["classify", "decisions", "extract"]) +def test_unsupported_system_instructions_are_rejected(config, model): + with pytest.raises(ScaleDownError, match="does not support system messages"): + config.transform_request( + f"scaledown/{model}", + [{"role": "system", "content": "instructions"}, {"role": "user", "content": "text"}], + {}, + {}, + {}, + ) + + +@respx.mock +@pytest.mark.parametrize("nested", [False, True]) +def test_classify_uses_native_labels_and_preserves_scores(nested): + payload = {"top_label": "urgent", "scores": {"urgent": 0.9, "routine": 0.1}, "input_tokens": 521} + route = respx.post(f"{BASE}/classify").mock(return_value=httpx.Response(200, json=payload)) + labels = [ + {"name": "urgent", "rubric": "Service is unavailable."}, + {"name": "routine", "rubric": "A cosmetic issue."}, + ] + options = {"extra_body": {"labels": labels}} if nested else {"labels": labels} + + response = litellm.completion( + model="scaledown/classify", + messages=[{"role": "user", "content": [{"type": "text", "text": "Server is down."}]}], + **options, + ) + + assert json.loads(route.calls[0].request.content) == {"text": "Server is down.", "labels": labels} + assert json.loads(response.choices[0].message.content) == payload + assert response.usage.prompt_tokens == 521 + assert litellm.completion_cost(completion_response=response) == pytest.approx( + 521 * litellm.model_cost["scaledown/classify"]["input_cost_per_token"] + ) + + +@respx.mock +@pytest.mark.parametrize("labels", [None, [], ["urgent"], [{"name": "urgent"}]]) +def test_classify_requires_labels_before_network(labels): + route = respx.post(f"{BASE}/classify") + with pytest.raises(litellm.BadRequestError, match=r"labels|rubric"): + litellm.completion( + model="scaledown/classify", messages=[{"role": "user", "content": "text"}], extra_body={"labels": labels} + ) + assert not route.called + + +@pytest.mark.asyncio +@respx.mock +async def test_async_extraction_keeps_flat_fields_and_usage(): + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + route = respx.post(f"{BASE}/extract").mock( + return_value=httpx.Response( + 200, + json={ + "entities": [{"type": "vendor", "text": "Northwind"}], + "structured_result": None, + "input_tokens": 161, + }, + ) + ) + client = AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()) + try: + response = await litellm.acompletion( + model="scaledown/extract", + messages=[{"role": "user", "content": "Invoice from Northwind."}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "invoice", + "strict": False, + "schema": {"properties": {"vendor": {"type": "string"}}}, + }, + }, + client=client, + ) + finally: + await client.close() + assert json.loads(response.choices[0].message.content) == {"vendor": "Northwind"} + assert response.usage.prompt_tokens == 161 + assert json.loads(route.calls[0].request.content) == { + "text": "Invoice from Northwind.", + "entities": {"vendor": "vendor"}, + } + + +@respx.mock +def test_router_stream_includes_usage_without_sending_stream_options_upstream(): + route = respx.post(f"{BASE}/summarization/abstractive").mock( + return_value=httpx.Response(200, json={"summary": "Fixed billing.", "input_tokens": 90}) + ) + router = litellm.Router( + model_list=[ + {"model_name": "summary", "litellm_params": {"model": "scaledown/summarize", "api_key": "test-key"}} + ], + num_retries=0, + ) + chunks = list( + router.completion( + model="summary", + messages=[{"role": "user", "content": "Billing was fixed on Monday."}], + stream=True, + stream_options={"include_usage": True}, + ) + ) + assert ( + json.loads("".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices))["summary"] + == "Fixed billing." + ) + assert chunks[-1].usage.prompt_tokens == 90 + assert json.loads(route.calls[0].request.content) == {"text": "Billing was fixed on Monday."} + + +@respx.mock +def test_pydantic_response_format_does_not_silently_claim_strict_support(): + from pydantic import BaseModel, ConfigDict + + class Invoice(BaseModel): + model_config = ConfigDict(frozen=True) + vendor: str + + route = respx.post(f"{BASE}/extract") + with pytest.raises(litellm.BadRequestError, match="strict JSON schemas"): + litellm.completion( + model="scaledown/extract", + messages=[{"role": "user", "content": "Invoice from Northwind."}], + response_format=Invoice, + ) + assert not route.called diff --git a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 63756ee108c..9f717f522c0 100644 --- a/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/unit/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -10794,12 +10794,12 @@ async def test_admission_request_body_serves_stashed_peek_callable(): from litellm.constants import MCP_PEEKED_BODY_SCOPE_KEY from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import _admission_request - jsonrpc_body = b'{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' + admission_body: Final = b'{"method":"tools/list"}' async def peek() -> bytes: - return jsonrpc_body + return admission_body - with_peek = _admission_request( + with_peek: Final = _admission_request( { "type": "http", "method": "POST", @@ -10808,7 +10808,9 @@ async def test_admission_request_body_serves_stashed_peek_callable(): MCP_PEEKED_BODY_SCOPE_KEY: peek, } ) - assert await with_peek.body() == jsonrpc_body + assert await with_peek.body() == admission_body - without_peek = _admission_request({"type": "http", "method": "POST", "path": "/mcp", "headers": []}) + without_peek: Final = _admission_request( + {"type": "http", "method": "POST", "path": "/mcp", "headers": []} + ) assert await without_peek.body() == b"{}" diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py index d046871c8e2..24b0f450d0b 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_max_concurrent_requests.py @@ -1,9 +1,15 @@ import asyncio -from typing import Dict, Optional +from collections.abc import Awaitable, Callable +from typing import Dict, Final, Optional, TypeVar, cast +from unittest.mock import AsyncMock, patch +from mcp import ClientSession +from mcp.server.context import ServerRequestContext +from mcp.types import CallToolRequestParams as MCPCallToolRequestParams +from litellm.experimental_mcp_client.client import MCPClient, PersistentMCPSession +from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var import pytest from mcp.types import CallToolResult, TextContent -from unittest.mock import patch from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.types.mcp import MCPAuth, MCPTransport @@ -12,6 +18,64 @@ from litellm.types.mcp_server.mcp_server_manager import MCPServer HOLD_SECONDS = 0.1 +_SessionResult = TypeVar("_SessionResult") + + +class _AdmissionProbeClient(MCPClient): + def __init__( + self, + server_id: str, + fingerprint: str, + tracker: "_ConcurrencyTracker", + submitted: asyncio.Event | None = None, + release: asyncio.Event | None = None, + ) -> None: + super().__init__(server_url="http://upstream/mcp", transport_type=MCPTransport.http) + self._server_id: Final = server_id + self._fingerprint: Final = fingerprint + self._tracker: Final = tracker + self._submitted: Final = submitted + self._release: Final = release + + async def discovery_auth_fingerprint(self) -> str: + return self._fingerprint + + async def run_with_session( + self, + operation: Callable[[ClientSession], Awaitable[_SessionResult]], + *, + quiet_on_error: bool = False, + on_stream_error: Callable[[asyncio.Future[Exception]], None] | None = None, + on_cleanup: Callable[[], None] | None = None, + ) -> _SessionResult: + del quiet_on_error, on_stream_error, on_cleanup + return await operation(cast(ClientSession, object())) + + async def _perform_call(self) -> CallToolResult: + self._tracker.enter(self._server_id) + try: + if self._release is not None: + await self._release.wait() + return CallToolResult(content=[], isError=False) + finally: + self._tracker.exit(self._server_id) + + async def call_tool( + self, + call_tool_request_params: MCPCallToolRequestParams, + host_progress_callback: Callable | None = None, + raise_on_error: bool = False, + allow_input_required: bool = False, + persistent_session: PersistentMCPSession | None = None, + ) -> CallToolResult: + del call_tool_request_params, host_progress_callback, raise_on_error, allow_input_required + if self._submitted is not None: + self._submitted.set() + if persistent_session is None: + return await self._perform_call() + return await persistent_session.run(lambda _: self._perform_call()) + + class _ConcurrencyTracker: """Records how many call_tool invocations are simultaneously in flight.""" @@ -32,14 +96,18 @@ class _ConcurrencyTracker: self.global_current -= 1 -def _make_server(server_id: str, max_concurrent_requests: Optional[int]) -> MCPServer: +def _make_server( + server_id: str, + max_concurrent_requests: Optional[int], + auth_type: MCPAuth = MCPAuth.none, +) -> MCPServer: return MCPServer( server_id=server_id, name=server_id, server_name=server_id, url="https://example.com", transport=MCPTransport.http, - auth_type=MCPAuth.none, + auth_type=auth_type, max_concurrent_requests=max_concurrent_requests, ) @@ -47,7 +115,13 @@ def _make_server(server_id: str, max_concurrent_requests: Optional[int]) -> MCPS def _patch_client_with_tracker(manager: MCPServerManager, tracker: _ConcurrencyTracker): async def fake_create_mcp_client(server, **kwargs): class _ProbeClient: - async def call_tool(self, params, host_progress_callback=None, allow_input_required=False): + async def call_tool( + self, + params, + host_progress_callback=None, + allow_input_required=False, + persistent_session: PersistentMCPSession | None = None, + ): tracker.enter(server.server_id) try: await asyncio.sleep(HOLD_SECONDS) @@ -79,6 +153,181 @@ async def _fire(manager: MCPServerManager, server: MCPServer, n: int) -> None: ) +@pytest.mark.parametrize("auth_type", [MCPAuth.none, MCPAuth.oauth2_token_exchange]) +@pytest.mark.asyncio +async def test_persistent_tool_call_with_limit_one_completes_without_nested_admission( + auth_type: MCPAuth, +) -> None: + manager: Final = MCPServerManager() + server: Final = _make_server("persistent-limit-one", 1, auth_type) + tracker: Final = _ConcurrencyTracker() + client: Final = _AdmissionProbeClient(server.server_id, "auth-fingerprint", tracker) + gateway_session_id: Final = "gw-limit-one" + oauth2_headers: Final = ( + {"Authorization": "Bearer subject-token"} if auth_type == MCPAuth.oauth2_token_exchange else None + ) + context_token: Final = active_mcp_request_ctx_var.set(cast(ServerRequestContext, object())) + manager.track_gateway_session(gateway_session_id) + try: + with patch.object(manager, "create_mcp_client", new=AsyncMock(return_value=client)): + response: Final = await asyncio.wait_for( + manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="tool", + arguments={}, + mcp_auth_header=None, + mcp_server_auth_headers=None, + oauth2_headers=oauth2_headers, + raw_headers={"mcp-session-id": gateway_session_id}, + proxy_logging_obj=None, + tasks=[], + ), + 1, + ) + assert response.is_error is False + finally: + sessions_to_close: Final = tuple(manager._upstream_sessions.values()) + manager.release_upstream_sessions(gateway_session_id) + await asyncio.gather(*(session.wait_closed() for session in sessions_to_close)) + active_mcp_request_ctx_var.reset(context_token) + + +@pytest.mark.asyncio +async def test_persistent_session_admission_uses_updated_server_limit() -> None: + manager: Final = MCPServerManager() + server: Final = _make_server("persistent-edited-limit", 2) + manager.registry = {server.server_id: server} + tracker: Final = _ConcurrencyTracker() + gateway_session_id: Final = "gw-edited-limit" + second_submitted: Final = asyncio.Event() + release_second: Final = asyncio.Event() + first_client: Final = _AdmissionProbeClient(server.server_id, "auth-fingerprint", tracker) + second_client: Final = _AdmissionProbeClient( + server.server_id, + "auth-fingerprint", + tracker, + submitted=second_submitted, + release=release_second, + ) + context_token: Final = active_mcp_request_ctx_var.set(cast(ServerRequestContext, object())) + manager.track_gateway_session(gateway_session_id) + + async def dispatch() -> CallToolResult: + return await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="tool", + arguments={}, + mcp_auth_header=None, + mcp_server_auth_headers=None, + oauth2_headers=None, + raw_headers={"mcp-session-id": gateway_session_id}, + proxy_logging_obj=None, + tasks=[], + ) + + try: + with patch.object(manager, "create_mcp_client", new=AsyncMock(side_effect=(first_client, second_client))): + first_response: Final = await asyncio.wait_for(dispatch(), 1) + assert first_response.is_error is False + + edited_server: Final = server.model_copy(update={"max_concurrent_requests": 1}) + manager.registry = {server.server_id: edited_server} + edited_semaphore: Final = manager._get_call_semaphore(edited_server) + assert edited_semaphore is not None + await edited_semaphore.acquire() + + second_call: Final = asyncio.create_task(dispatch()) + try: + await asyncio.wait_for(second_submitted.wait(), 1) + await asyncio.sleep(HOLD_SECONDS) + assert not second_call.done() + assert tracker.current_by_server.get(server.server_id, 0) == 0 + finally: + edited_semaphore.release() + release_second.set() + await asyncio.gather(second_call, return_exceptions=True) + + second_response: Final = await asyncio.wait_for(second_call, 1) + assert second_response.is_error is False + finally: + release_second.set() + sessions_to_close: Final = tuple(manager._upstream_sessions.values()) + tuple( + session for _, session in manager._retiring_upstream_sessions + ) + manager.release_upstream_sessions(gateway_session_id) + await asyncio.gather(*(session.wait_closed() for session in sessions_to_close), return_exceptions=True) + active_mcp_request_ctx_var.reset(context_token) + + +@pytest.mark.asyncio +async def test_queued_persistent_call_survives_header_change_while_waiting_for_admission() -> None: + manager: Final = MCPServerManager() + server: Final = _make_server("persistent-header-change", 1) + tracker: Final = _ConcurrencyTracker() + gateway_session_id: Final = "gw-header-change" + holder_started: Final = asyncio.Event() + release_holder: Final = asyncio.Event() + first_submitted: Final = asyncio.Event() + second_submitted: Final = asyncio.Event() + first_client: Final = _AdmissionProbeClient( + server.server_id, "first-fingerprint", tracker, submitted=first_submitted + ) + second_client: Final = _AdmissionProbeClient( + server.server_id, "second-fingerprint", tracker, submitted=second_submitted + ) + context_token: Final = active_mcp_request_ctx_var.set(cast(ServerRequestContext, object())) + manager.track_gateway_session(gateway_session_id) + + async def hold_slot() -> None: + async with manager._limit_outbound_concurrency(server): + holder_started.set() + await release_holder.wait() + + async def dispatch(authorization: str) -> CallToolResult: + return await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="tool", + arguments={}, + mcp_auth_header=None, + mcp_server_auth_headers=None, + oauth2_headers=None, + raw_headers={"mcp-session-id": gateway_session_id, "Authorization": authorization}, + proxy_logging_obj=None, + tasks=[], + ) + + holder_task: Final = asyncio.create_task(hold_slot()) + call_tasks: Final[list[asyncio.Task[CallToolResult]]] = [] + try: + await asyncio.wait_for(holder_started.wait(), 1) + with patch.object(manager, "create_mcp_client", new=AsyncMock(side_effect=(first_client, second_client))): + call_tasks.append(asyncio.create_task(dispatch("Bearer old"))) + await asyncio.wait_for(first_submitted.wait(), 1) + old_session: Final = next(iter(manager._upstream_sessions.values())) + call_tasks.append(asyncio.create_task(dispatch("Bearer new"))) + await asyncio.wait_for(second_submitted.wait(), 1) + + assert (gateway_session_id, old_session) in manager._retiring_upstream_sessions + assert not old_session.closed + assert tracker.current_by_server.get(server.server_id, 0) == 0 + + release_holder.set() + responses: Final = await asyncio.wait_for(asyncio.gather(*call_tasks), 1) + + assert all(response.is_error is False for response in responses) + await asyncio.wait_for(old_session.wait_closed(), 1) + finally: + release_holder.set() + sessions_to_close: Final = tuple( + manager._upstream_sessions.values() + ) + tuple(session for _, session in manager._retiring_upstream_sessions) + manager.release_upstream_sessions(gateway_session_id) + await asyncio.gather(*(session.wait_closed() for session in sessions_to_close), return_exceptions=True) + await asyncio.gather(*call_tasks, return_exceptions=True) + await holder_task + active_mcp_request_ctx_var.reset(context_token) + + @pytest.mark.asyncio async def test_max_concurrent_requests_caps_in_flight_tool_calls(): """A configured cap of 2 must never let more than 2 calls hit one server at once.""" diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py index 3953d9d086c..3345683517b 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py @@ -1,5 +1,6 @@ # Create server parameters for stdio connection import asyncio +import json import os from typing import Final @@ -7,6 +8,8 @@ import pytest from unittest.mock import ANY, AsyncMock, MagicMock, patch from contextlib import asynccontextmanager, nullcontext +from starlette.types import Receive, Scope, Send + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( MCPServerManager, @@ -3134,7 +3137,8 @@ async def test_mcp_admission_body_peek_times_out_without_losing_the_body(monkeyp @pytest.mark.asyncio -async def test_mcp_routing_stashes_peeked_body_for_auth(): +@pytest.mark.parametrize("method", ["tools/list", "tools/call", ["tools/list"]]) +async def test_mcp_routing_stashes_only_method_for_auth_and_replays_full_body(method: str | list[str]): try: from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCP_PEEKED_BODY_SCOPE_KEY, @@ -3149,7 +3153,15 @@ async def test_mcp_routing_stashes_peeked_body_for_auth(): except ImportError: pytest.skip("MCP server not available") - jsonrpc_body: Final = b'{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}' + jsonrpc_body: Final = json.dumps( + { + "jsonrpc": "2.0", + "id": 2, + "method": method, + "model": "zero-cost-model", + "params": {}, + } + ).encode() scope = { "type": "http", "method": "POST", @@ -3161,10 +3173,15 @@ async def test_mcp_routing_stashes_peeked_body_for_auth(): } receive = AsyncMock(side_effect=[{"type": "http.request", "body": jsonrpc_body, "more_body": False}]) send = AsyncMock() - stateless_called = [] - async def stateless_handle(s, r, se) -> None: - stateless_called.append(1) # mutable-ok: records the manager the handler dispatched to + async def stateless_handle(_scope: Scope, downstream_receive: Receive, _send: Send) -> None: + assert await downstream_receive() == { + "type": "http.request", + "body": jsonrpc_body, + "more_body": False, + } + + stateless_handle_mock: Final = AsyncMock(side_effect=stateless_handle) with ( patch( # test-quality-ok: the ASGI handler reads auth from a module-level helper; the suite's only seam @@ -3182,7 +3199,7 @@ async def test_mcp_routing_stashes_peeked_body_for_auth(): True, ), patch.object( # test-quality-ok: session managers are module-level singletons; the suite's only seam - session_manager_stateless, "handle_request", side_effect=stateless_handle + session_manager_stateless, "handle_request", side_effect=stateless_handle_mock ), patch.object( # test-quality-ok: session managers are module-level singletons; the suite's only seam session_manager_stateless, "_server_instances", {} @@ -3193,10 +3210,10 @@ async def test_mcp_routing_stashes_peeked_body_for_auth(): ): await handle_streamable_http_mcp(scope, receive, send) - assert stateless_called, "tools/list without a session should route to the stateless manager" - assert await scope[MCP_PEEKED_BODY_SCOPE_KEY]() == jsonrpc_body - request_data: Final = await _read_request_body(_admission_request(scope)) - assert request_data.get("method") == "tools/list" + stateless_handle_mock.assert_awaited_once() + assert await _read_request_body(_admission_request(scope)) == ( + {"method": method} if isinstance(method, str) else {} + ) @pytest.mark.asyncio diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py index c1d64bf3d3a..e670c9980a4 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -7,10 +7,11 @@ import logging import os import sys import time -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from datetime import datetime from pathlib import Path -from typing import Any, Dict, Final, Literal, Optional +from types import SimpleNamespace +from typing import Any, Dict, Final, Literal, Optional, TypeVar, cast from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -30,8 +31,10 @@ import contextlib import httpx import httpx2 -from mcp import ReadResourceResult, Resource +from mcp import ClientSession, ReadResourceResult, Resource +from mcp.server.context import ServerRequestContext from mcp.types import ( + CallToolRequestParams as MCPCallToolRequestParams, CallToolResult, GetPromptResult, Prompt, @@ -43,7 +46,10 @@ from pydantic import AnyUrl, TypeAdapter from litellm.constants import MCP_METADATA_TIMEOUT from litellm.proxy._experimental.mcp_server import discoverable_endpoints +from litellm.proxy._experimental.mcp_server.legacy_callbacks import ElicitationCallback +from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var from litellm.proxy._experimental.mcp_server.tool_outcome import TextResult +from litellm.experimental_mcp_client.client import MCPClient, PersistentMCPSession from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( ListedToolsCaller, MCPServerManager, @@ -81,6 +87,8 @@ from litellm.proxy.utils import ProxyLogging from litellm.types.guardrails import GuardrailEventHooks from litellm.types.integrations.slack_alerting import AlertType +_TSessionResult = TypeVar("_TSessionResult") + @pytest.mark.asyncio async def test_manager_sampling_preserves_explicit_headers_without_ambient_context(): @@ -7170,7 +7178,12 @@ class TestMCPServerManager: # Create mock client that tracks call_tool usage mock_client = AsyncMock() - async def mock_call_tool(params, host_progress_callback=None, allow_input_required=False): + async def mock_call_tool( + params, + host_progress_callback=None, + allow_input_required=False, + persistent_session: PersistentMCPSession | None = None, + ): # Return a mock CallToolResult result = MagicMock(spec=CallToolResult) result.content = [{"type": "text", "text": "Tool executed successfully"}] @@ -11618,9 +11631,18 @@ class _RetryFakeClient: self._result = result self._MCPClient = MCPClient self.attempts = 0 + self.persistent_sessions: list[PersistentMCPSession | None] = [] - async def call_tool(self, params, host_progress_callback=None, raise_on_error=False, allow_input_required=False): + async def call_tool( + self, + params, + host_progress_callback=None, + raise_on_error=False, + allow_input_required=False, + persistent_session: PersistentMCPSession | None = None, + ): self.attempts += 1 + self.persistent_sessions.append(persistent_session) if self._raises is not None: if raise_on_error: raise self._raises @@ -11651,11 +11673,12 @@ class TestOBOCallToolRetry: return manager @pytest.mark.asyncio - async def test_upstream_401_invalidates_and_retries_once(self): + async def test_upstream_401_on_persistent_session_invalidates_and_retries_once(self): manager = self._manager() - success = CallToolResult(content=[], isError=False) - first = _RetryFakeClient(raises=_UpstreamAuthError(401)) - retry = _RetryFakeClient(result=success) + success: Final = CallToolResult(content=[], isError=False) + first: Final = _RetryFakeClient(raises=_UpstreamAuthError(401)) + retry: Final = _RetryFakeClient(result=success) + persistent_session: Final = MagicMock() manager.create_mcp_client = AsyncMock(return_value=retry) result = await manager._obo_call_tool_with_retry( @@ -11668,12 +11691,14 @@ class TestOBOCallToolRetry: stdio_env=None, subject_token="caller-jwt", user_api_key_auth=None, + persistent_session=persistent_session, ) assert result is success manager._cred_provider.invalidate_credentials.assert_awaited_once() manager.create_mcp_client.assert_awaited_once() assert first.attempts == 1 and retry.attempts == 1 + assert first.persistent_sessions == [persistent_session] @pytest.mark.asyncio async def test_upstream_401_on_id_jag_evicts_the_cached_bearer_and_retries(self): @@ -11802,6 +11827,68 @@ class TestOBOCallToolRetry: manager.create_mcp_client.assert_awaited_once() assert first.attempts == 1 and retry.attempts == 1 + @pytest.mark.asyncio + async def test_session_closed_by_a_peers_refresh_retries_on_a_fresh_session_without_invalidating(self) -> None: + from litellm.experimental_mcp_client.client import UpstreamSessionClosedError + + manager: Final = self._manager() + success: Final = CallToolResult(content=[], isError=False) + first: Final = _RetryFakeClient(raises=UpstreamSessionClosedError()) + retry: Final = _RetryFakeClient(result=success) + manager.create_mcp_client = AsyncMock(return_value=retry) + + result: Final = await manager._obo_call_tool_with_retry( + client=first, + call_tool_params=MagicMock(), + host_progress_callback=None, + mcp_server=_obo_server(), + server_auth_header=None, + extra_headers=None, + stdio_env=None, + subject_token="caller-jwt", + user_api_key_auth=None, + ) + + assert result is success + manager._cred_provider.invalidate_credentials.assert_not_awaited() + assert first.attempts == 1 and retry.attempts == 1 + + @pytest.mark.asyncio + async def test_a_401_evicts_the_cached_token_before_releasing_peers_on_the_shared_session(self) -> None: + """Peers woken by the shared session closing rebuild their client at once, so the stale token + must already be gone from the cache when the session closes or they re-exchange the same token.""" + manager: Final = self._manager() + order: Final[list[str]] = [] + + async def slow_invalidate(*_: object) -> None: + await asyncio.sleep(0.01) + order.append("invalidate") + + manager._cred_provider.invalidate_credentials = AsyncMock(side_effect=slow_invalidate) + shared: Final = MagicMock() + shared.close = MagicMock(side_effect=lambda: order.append("close")) + manager._upstream_sessions[("gw", "obo-srv", "fp")] = shared + first: Final = _RetryFakeClient(raises=_UpstreamAuthError(401)) + retry: Final = _RetryFakeClient(result=CallToolResult(content=[], isError=False)) + manager.create_mcp_client = AsyncMock(return_value=retry) + manager._upstream_session_for = AsyncMock(return_value=None) + + await manager._obo_call_tool_with_retry( + client=first, + call_tool_params=MagicMock(), + host_progress_callback=None, + mcp_server=_obo_server(), + server_auth_header=None, + extra_headers=None, + stdio_env=None, + subject_token="caller-jwt", + user_api_key_auth=None, + persistent_session=shared, + ) + + assert order == ["invalidate", "close"], order + assert ("gw", "obo-srv", "fp") not in manager._upstream_sessions + class TestOBOConcurrencyLimit: """OBO (token_exchange) tool calls must honor the server's max_concurrent_requests. @@ -11815,7 +11902,7 @@ class TestOBOConcurrencyLimit: async def test_obo_dispatch_respects_max_concurrent_requests(self): max_concurrent = 2 overflow = 3 - server = MCPServer( + server: Final = MCPServer( server_id="obo-concurrency", name="obo", url="https://upstream.example/mcp", @@ -11832,7 +11919,12 @@ class TestOBOConcurrencyLimit: class _ConcurrencyRecordingClient: async def call_tool( - self, params, host_progress_callback=None, raise_on_error=False, allow_input_required=False + self, + params, + host_progress_callback=None, + raise_on_error=False, + allow_input_required=False, + persistent_session: PersistentMCPSession | None = None, ): inflight["current"] += 1 inflight["peak"] = max(inflight["peak"], inflight["current"]) @@ -11842,7 +11934,7 @@ class TestOBOConcurrencyLimit: inflight["current"] -= 1 return CallToolResult(content=[], isError=False) - manager = MCPServerManager() + manager: Final = MCPServerManager() manager.create_mcp_client = AsyncMock(return_value=_ConcurrencyRecordingClient()) async def _dispatch(): @@ -11881,6 +11973,67 @@ class TestOBOConcurrencyLimit: assert inflight["current"] == 0 assert all(result.is_error is False for result in results) + @pytest.mark.asyncio + async def test_obo_dispatch_reuses_the_gateway_sessions_persistent_upstream_session( + self, _active_mcp_request_context: None + ) -> None: + server: Final = MCPServer( + server_id="obo-stateful", + name="obo", + url="https://upstream.example/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2_token_exchange, + token_exchange_endpoint="https://idp.example.com/token", + client_id="cid", + client_secret="csec", + ) + sessions_seen: Final[list[PersistentMCPSession | None]] = [] + + class _SessionRecordingClient: + async def discovery_auth_fingerprint(self) -> str: + return "same-token" + + def session_settings(self) -> tuple[MCPTransport, MCPUpstreamProtocol, float, bool, bool, bool]: + return (MCPTransport.http, "2025-06-18", 30.0, False, False, False) + + def open_persistent_session( + self, + admission: Callable[[], contextlib.AbstractAsyncContextManager[None]] | None = None, + ) -> SimpleNamespace: + return SimpleNamespace(closed=False, reusable=True, close=lambda: None) + + async def call_tool( + self, + params: MCPCallToolRequestParams, + host_progress_callback: Callable[..., Awaitable[None]] | None = None, + raise_on_error: bool = False, + allow_input_required: bool = False, + persistent_session: PersistentMCPSession | None = None, + ) -> CallToolResult: + sessions_seen.append(persistent_session) + return CallToolResult(content=[], isError=False) + + manager: Final = MCPServerManager() + manager.create_mcp_client = AsyncMock(return_value=_SessionRecordingClient()) + manager.track_gateway_session("gateway-1") + + for tool in ("select_project", "create_feature"): + result: Final = await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name=tool, + arguments={}, + tasks=[], + mcp_auth_header=None, + mcp_server_auth_headers=None, + oauth2_headers={"Authorization": "Bearer subject-jwt"}, + raw_headers={"mcp-session-id": "gateway-1"}, + proxy_logging_obj=None, + ) + assert result.is_error is False + + assert len(sessions_seen) == 2 and None not in sessions_seen, sessions_seen + assert sessions_seen[0] is sessions_seen[1], "OBO calls in one gateway session must share one upstream session" + class TestOBOEndpointDiscovery: """An oauth2_token_exchange server with no configured token endpoint discovers it (RFC 9728 -> @@ -17755,6 +17908,574 @@ async def test_catalog_waiter_that_observed_newer_revision_refreshes(monkeypatch assert reads[0] == 3 +class _OfflineMCPClient(MCPClient): + """An MCPClient whose sessions never touch the network: operations run against a stand-in session.""" + + async def run_with_session( + self, + operation: Callable[[ClientSession], Awaitable[_TSessionResult]], + *, + quiet_on_error: bool = False, + on_stream_error: Callable[[asyncio.Future[Exception]], None] | None = None, + on_cleanup: Callable[[], None] | None = None, + ) -> _TSessionResult: + del quiet_on_error, on_stream_error, on_cleanup + return await operation(cast(ClientSession, object())) + + +class _RetiringSessionStub: + def __init__(self) -> None: + self.retired = False + self.closed = False + self.reusable = True + + def retire(self) -> None: + self.retired = True + self.reusable = False + + def close(self) -> None: + self.closed = True + + async def wait_closed(self) -> None: + return None + + +class _RetirementRecordingClient(_OfflineMCPClient): + def __init__( + self, + server_url: str, + session: _RetiringSessionStub, + extra_headers: dict[str, str], + ) -> None: + super().__init__( + server_url=server_url, + transport_type=MCPTransport.http, + extra_headers=extra_headers, + ) + self._session = cast(PersistentMCPSession, session) + + def open_persistent_session( + self, + admission: Callable[[], contextlib.AbstractAsyncContextManager[None]] | None = None, + ) -> PersistentMCPSession: + return self._session + + +class _BlockingFingerprintClient(_OfflineMCPClient): + def __init__( + self, + server_url: str, + fingerprint_started: asyncio.Event, + resume_fingerprint: asyncio.Event, + session: _RetiringSessionStub, + ) -> None: + super().__init__(server_url=server_url, transport_type=MCPTransport.http) + self._fingerprint_started = fingerprint_started + self._resume_fingerprint = resume_fingerprint + self._session = cast(PersistentMCPSession, session) + self.open_persistent_session_calls = 0 + + async def discovery_auth_fingerprint(self) -> str: + self._fingerprint_started.set() + await self._resume_fingerprint.wait() + return "fingerprint" + + def open_persistent_session( + self, + admission: Callable[[], contextlib.AbstractAsyncContextManager[None]] | None = None, + ) -> PersistentMCPSession: + self.open_persistent_session_calls += 1 + return self._session + + +@pytest.mark.asyncio +async def test_dropped_upstream_session_stays_tracked_until_gateway_release() -> None: + manager: Final = MCPServerManager() + gateway_session_id: Final = "gw-dropped" + client: Final = _OfflineMCPClient(server_url="http://upstream/mcp", transport_type=MCPTransport.http) + session_stub: Final = _RetiringSessionStub() + session: Final = cast(PersistentMCPSession, session_stub) + key: Final = ( + gateway_session_id, + "s1", + await client.discovery_auth_fingerprint(), + client.session_settings(), + ) + manager.track_gateway_session(gateway_session_id) + manager._upstream_sessions[key] = session + + manager._drop_upstream_session(session) + + assert session_stub.closed + assert manager._upstream_sessions == {} + assert manager._retiring_upstream_sessions == [(gateway_session_id, session)] + + manager.release_upstream_sessions(gateway_session_id) + + assert manager._retiring_upstream_sessions == [] + + +@pytest.mark.asyncio +async def test_unreusable_upstream_session_is_replaced_before_cleanup_finishes( + _active_mcp_request_context: None, +) -> None: + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="s1", name="stateful", url="http://upstream/mcp", transport=MCPTransport.http) + gateway_session_id: Final = "gw-ending-session" + gateway_session_headers: Final = {"mcp-session-id": gateway_session_id} + ending_stub: Final = _RetiringSessionStub() + ending_stub.reusable = False + replacement_stub: Final = _RetiringSessionStub() + client: Final = _RetirementRecordingClient(server.url, replacement_stub, {}) + manager.track_gateway_session(gateway_session_id) + key: Final = ( + gateway_session_id, + server.server_id, + await client.discovery_auth_fingerprint(), + client.session_settings(), + ) + ending_session: Final = cast(PersistentMCPSession, ending_stub) + replacement_session: Final = cast(PersistentMCPSession, replacement_stub) + manager._upstream_sessions[key] = ending_session + + try: + session: Final = await manager._upstream_session_for(client, server, gateway_session_headers) + + assert session is replacement_session + assert ending_stub.retired + assert not ending_stub.closed + assert manager._retiring_upstream_sessions == [(gateway_session_id, ending_session)] + finally: + manager.release_upstream_sessions(gateway_session_id) + + +@pytest.fixture +def _active_mcp_request_context() -> Iterator[None]: + token: Final = active_mcp_request_ctx_var.set(cast(ServerRequestContext, object())) + try: + yield + finally: + active_mcp_request_ctx_var.reset(token) + + +@pytest.mark.asyncio +async def test_release_during_auth_fingerprinting_does_not_open_upstream_session( + _active_mcp_request_context: None, +) -> None: + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="s1", name="stateful", url="http://upstream/mcp", transport=MCPTransport.http) + gateway_session_id: Final = "gw-release-during-fingerprint" + gateway_session_headers: Final = {"mcp-session-id": gateway_session_id} + fingerprint_started: Final = asyncio.Event() + resume_fingerprint: Final = asyncio.Event() + client: Final = _BlockingFingerprintClient( + server.url, + fingerprint_started, + resume_fingerprint, + _RetiringSessionStub(), + ) + manager.track_gateway_session(gateway_session_id) + request_task: Final = asyncio.create_task( + manager._upstream_session_for(client, server, gateway_session_headers) + ) + + try: + await asyncio.wait_for(fingerprint_started.wait(), 5) + manager.release_upstream_sessions(gateway_session_id) + resume_fingerprint.set() + + session: Final = await asyncio.wait_for(request_task, 5) + + assert session is None + assert client.open_persistent_session_calls == 0 + assert not any(key[0] == gateway_session_id for key in manager._upstream_sessions) + finally: + resume_fingerprint.set() + manager.release_upstream_sessions(gateway_session_id) + if not request_task.done(): + await asyncio.wait_for(request_task, 5) + + +@pytest.mark.asyncio +async def test_upstream_session_is_shared_per_gateway_session_and_released_with_it( + _active_mcp_request_context: None, +) -> None: + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="s1", name="stateful", url="http://upstream/mcp", transport=MCPTransport.http) + client: Final = _OfflineMCPClient(server_url=server.url, transport_type=MCPTransport.http) + try: + assert await manager._upstream_session_for(client, server, None) is None + assert await manager._upstream_session_for(client, server, {"accept": "application/json"}) is None + assert await manager._upstream_session_for(client, server, {"mcp-session-id": "forged"}) is None, ( + "an mcp-session-id the gateway never issued must not open a long-lived upstream session" + ) + + manager.track_gateway_session("gw-1") + manager.track_gateway_session("gw-2") + first: Final = await manager._upstream_session_for(client, server, {"Mcp-Session-Id": "gw-1"}) + second: Final = await manager._upstream_session_for(client, server, {"mcp-session-id": "gw-1"}) + other: Final = await manager._upstream_session_for(client, server, {"mcp-session-id": "gw-2"}) + assert first is not None and first is second, "every call of one gateway session must share one upstream session" + assert other is not None and other is not first, "distinct gateway sessions must not share upstream state" + + manager.release_upstream_sessions("gw-1") + await asyncio.wait_for(first.wait_closed(), 5) + assert first.closed and not other.closed + assert await manager._upstream_session_for(client, server, {"mcp-session-id": "gw-1"}) is None, ( + "a released gateway session must not reopen upstream sessions" + ) + manager.track_gateway_session("gw-1") + replacement: Final = await manager._upstream_session_for(client, server, {"mcp-session-id": "gw-1"}) + assert replacement is not first and not replacement.closed + + other.close() + await asyncio.wait_for(other.wait_closed(), 5) + reopened: Final = await manager._upstream_session_for(client, server, {"mcp-session-id": "gw-2"}) + assert reopened is not other and not reopened.closed, "a dead upstream session must be replaced, not reused" + + stdio: Final = MCPServer(server_id="s2", name="local", command="cat", transport=MCPTransport.stdio) + assert await manager._upstream_session_for(client, stdio, {"mcp-session-id": "gw-1"}) is None + finally: + for gateway_session_id in ("gw-1", "gw-2"): + manager.release_upstream_sessions(gateway_session_id) + await asyncio.wait_for( + asyncio.gather(*(s.wait_closed() for s in manager._upstream_sessions.values()), return_exceptions=True), 5 + ) + + +@pytest.mark.asyncio +async def test_upstream_sessions_are_partitioned_by_discovery_auth_fingerprint( + _active_mcp_request_context: None, +) -> None: + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="s1", name="stateful", url="http://upstream/mcp", transport=MCPTransport.http) + first_client: Final = _OfflineMCPClient( + server_url=server.url, + transport_type=MCPTransport.http, + extra_headers={"Authorization": "Bearer first"}, + ) + second_client: Final = _OfflineMCPClient( + server_url=server.url, + transport_type=MCPTransport.http, + extra_headers={"Authorization": "Bearer second"}, + ) + first_fingerprint: Final = await first_client.discovery_auth_fingerprint() + second_fingerprint: Final = await second_client.discovery_auth_fingerprint() + assert first_fingerprint != second_fingerprint + + gateway_session_headers: Final = {"mcp-session-id": "gw"} + manager.track_gateway_session("gw") + try: + first_session: Final = await manager._upstream_session_for(first_client, server, gateway_session_headers) + second_session: Final = await manager._upstream_session_for(second_client, server, gateway_session_headers) + assert first_session is not None + assert second_session is not None + assert first_session is not second_session + await asyncio.wait_for(first_session.wait_closed(), 5) + assert first_session.closed + assert len(manager._upstream_sessions) == 1 + assert tuple(manager._upstream_sessions.values()) == (second_session,) + first_session_again: Final = await manager._upstream_session_for( + first_client, server, gateway_session_headers + ) + assert first_session_again is not None + assert first_session_again is not first_session + assert first_session_again is not second_session + await asyncio.wait_for(second_session.wait_closed(), 5) + assert second_session.closed + assert len(manager._upstream_sessions) == 1 + assert tuple(manager._upstream_sessions.values()) == (first_session_again,) + finally: + sessions: Final = tuple(manager._upstream_sessions.values()) + manager.release_upstream_sessions("gw") + await asyncio.wait_for(asyncio.gather(*(session.wait_closed() for session in sessions)), 5) + + +@pytest.mark.parametrize("changed_setting", ["protocol_version", "elicitation_callback"]) +@pytest.mark.asyncio +async def test_upstream_sessions_are_partitioned_by_connection_settings( + changed_setting: Literal["protocol_version", "elicitation_callback"], + _active_mcp_request_context: None, +) -> None: + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="s1", name="stateful", url="http://upstream/mcp", transport=MCPTransport.http) + first_client: Final = _OfflineMCPClient(server_url=server.url, transport_type=MCPTransport.http) + changed_client: Final = ( + _OfflineMCPClient( + server_url=server.url, + transport_type=MCPTransport.http, + protocol_version="2025-06-18", + ) + if changed_setting == "protocol_version" + else _OfflineMCPClient( + server_url=server.url, + transport_type=MCPTransport.http, + elicitation_callback=cast(ElicitationCallback, AsyncMock()), + ) + ) + third_client: Final = _OfflineMCPClient(server_url=server.url, transport_type=MCPTransport.http) + gateway_session_headers: Final = {"mcp-session-id": "gw-settings"} + manager.track_gateway_session("gw-settings") + + try: + assert await first_client.discovery_auth_fingerprint() == await changed_client.discovery_auth_fingerprint() + first_session: Final = await manager._upstream_session_for(first_client, server, gateway_session_headers) + changed_session: Final = await manager._upstream_session_for(changed_client, server, gateway_session_headers) + assert first_session is not None + assert changed_session is not None + assert first_session is not changed_session + await asyncio.wait_for(first_session.wait_closed(), 5) + assert first_session.closed + assert len(manager._upstream_sessions) == 1 + assert tuple(manager._upstream_sessions.values()) == (changed_session,) + first_session_again: Final = await manager._upstream_session_for( + third_client, server, gateway_session_headers + ) + + assert first_session_again is not None + assert first_session_again is not first_session + assert first_session_again is not changed_session + await asyncio.wait_for(changed_session.wait_closed(), 5) + assert changed_session.closed + assert len(manager._upstream_sessions) == 1 + assert tuple(manager._upstream_sessions.values()) == (first_session_again,) + finally: + sessions: Final = tuple(manager._upstream_sessions.values()) + manager.release_upstream_sessions("gw-settings") + await asyncio.wait_for(asyncio.gather(*(session.wait_closed() for session in sessions)), 5) + + +@pytest.mark.asyncio +async def test_upstream_session_registry_stays_bounded_when_forwarded_authorization_changes( + _active_mcp_request_context: None, +) -> None: + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="s1", name="stateful", url="http://upstream/mcp", transport=MCPTransport.http) + clients: Final = tuple( + _OfflineMCPClient( + server_url=server.url, + transport_type=MCPTransport.http, + extra_headers={"Authorization": f"Bearer token-{index}"}, + ) + for index in range(5) + ) + gateway_session_headers: Final = {"mcp-session-id": "gw-churn"} + manager.track_gateway_session("gw-churn") + sessions: Final = tuple( + [await manager._upstream_session_for(client, server, gateway_session_headers) for client in clients] + ) + assert all(session is not None for session in sessions) + typed_sessions: Final = cast(tuple[PersistentMCPSession, ...], sessions) + + try: + assert len(manager._upstream_sessions) == 1 + assert tuple(manager._upstream_sessions.values()) == (typed_sessions[-1],) + await asyncio.wait_for( + asyncio.gather(*(session.wait_closed() for session in typed_sessions[:-1])), + 5, + ) + + assert all(session.closed for session in typed_sessions[:-1]) + finally: + manager.release_upstream_sessions("gw-churn") + await asyncio.wait_for( + asyncio.gather(*(session.wait_closed() for session in typed_sessions), return_exceptions=True), + 5, + ) + + +@pytest.mark.asyncio +async def test_release_upstream_sessions_closes_retiring_sessions(_active_mcp_request_context: None) -> None: + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="s1", name="stateful", url="http://upstream/mcp", transport=MCPTransport.http) + first_stub: Final = _RetiringSessionStub() + second_stub: Final = _RetiringSessionStub() + first_client: Final = _RetirementRecordingClient( + server.url, + first_stub, + {"Authorization": "Bearer first"}, + ) + second_client: Final = _RetirementRecordingClient( + server.url, + second_stub, + {"Authorization": "Bearer second"}, + ) + gateway_session_id: Final = "gw-release-retiring" + gateway_session_headers: Final = {"mcp-session-id": gateway_session_id} + manager.track_gateway_session(gateway_session_id) + + first_session: Final = await manager._upstream_session_for(first_client, server, gateway_session_headers) + second_session: Final = await manager._upstream_session_for(second_client, server, gateway_session_headers) + + assert first_session is cast(PersistentMCPSession, first_stub) + assert second_session is cast(PersistentMCPSession, second_stub) + assert first_stub.retired and not first_stub.closed + assert manager._retiring_upstream_sessions == [(gateway_session_id, first_session)] + + manager.release_upstream_sessions(gateway_session_id) + + assert first_stub.closed + assert second_stub.closed + assert manager._retiring_upstream_sessions == [] + + +@pytest.mark.asyncio +async def test_close_all_upstream_sessions_closes_retiring_sessions(_active_mcp_request_context: None) -> None: + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="s1", name="stateful", url="http://upstream/mcp", transport=MCPTransport.http) + first_stub: Final = _RetiringSessionStub() + second_stub: Final = _RetiringSessionStub() + first_client: Final = _RetirementRecordingClient( + server.url, + first_stub, + {"Authorization": "Bearer first"}, + ) + second_client: Final = _RetirementRecordingClient( + server.url, + second_stub, + {"Authorization": "Bearer second"}, + ) + gateway_session_id: Final = "gw-shutdown-retiring" + gateway_session_headers: Final = {"mcp-session-id": gateway_session_id} + manager.track_gateway_session(gateway_session_id) + + first_session: Final = await manager._upstream_session_for(first_client, server, gateway_session_headers) + second_session: Final = await manager._upstream_session_for(second_client, server, gateway_session_headers) + + assert first_session is not None and second_session is not None + assert first_stub.retired and not first_stub.closed + assert manager._retiring_upstream_sessions == [(gateway_session_id, first_session)] + + await manager.close_all_upstream_sessions() + + assert first_stub.closed + assert second_stub.closed + assert manager._upstream_sessions == {} + assert manager._retiring_upstream_sessions == [] + assert manager._live_gateway_sessions == frozenset() + + +@pytest.mark.asyncio +async def test_closed_retiring_sessions_are_pruned_on_next_eviction(_active_mcp_request_context: None) -> None: + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="s1", name="stateful", url="http://upstream/mcp", transport=MCPTransport.http) + stubs: Final = tuple(_RetiringSessionStub() for _ in range(3)) + clients: Final = tuple( + _RetirementRecordingClient( + server.url, + stub, + {"Authorization": f"Bearer token-{index}"}, + ) + for index, stub in enumerate(stubs) + ) + gateway_session_id: Final = "gw-prune-retiring" + gateway_session_headers: Final = {"mcp-session-id": gateway_session_id} + manager.track_gateway_session(gateway_session_id) + + first_session: Final = await manager._upstream_session_for(clients[0], server, gateway_session_headers) + second_session: Final = await manager._upstream_session_for(clients[1], server, gateway_session_headers) + assert first_session is not None and second_session is not None + assert manager._retiring_upstream_sessions == [(gateway_session_id, first_session)] + + stubs[0].closed = True + third_session: Final = await manager._upstream_session_for(clients[2], server, gateway_session_headers) + + assert third_session is not None + assert manager._retiring_upstream_sessions == [(gateway_session_id, second_session)] + assert all(session is not first_session for _, session in manager._retiring_upstream_sessions) + + manager.release_upstream_sessions(gateway_session_id) + + +@pytest.mark.asyncio +async def test_close_all_upstream_sessions_closes_sessions_and_clears_registries( + _active_mcp_request_context: None, +) -> None: + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="s1", name="stateful", url="http://upstream/mcp", transport=MCPTransport.http) + client: Final = _OfflineMCPClient(server_url=server.url, transport_type=MCPTransport.http) + manager.track_gateway_session("gw-1") + manager.track_gateway_session("gw-2") + + try: + first_session: Final = await manager._upstream_session_for(client, server, {"mcp-session-id": "gw-1"}) + second_session: Final = await manager._upstream_session_for(client, server, {"mcp-session-id": "gw-2"}) + assert first_session is not None and second_session is not None + sessions: Final = (first_session, second_session) + + await manager.close_all_upstream_sessions() + + assert all(session.closed for session in sessions) + assert manager._upstream_sessions == {} + assert manager._live_gateway_sessions == frozenset() + finally: + sessions_to_close: Final = tuple(manager._upstream_sessions.values()) + for session in sessions_to_close: + session.close() + await asyncio.wait_for(asyncio.gather(*(session.wait_closed() for session in sessions_to_close)), 5) + + +@pytest.mark.asyncio +async def test_close_all_upstream_sessions_logs_when_waiting_times_out( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + class NeverClosingSession: + def __init__(self) -> None: + self._never_close: Final = asyncio.Event() + self.closed = False + + def close(self) -> None: + self.closed = True + + async def wait_closed(self) -> None: + await self._never_close.wait() + + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_UPSTREAM_SESSION_SHUTDOWN_TIMEOUT_SECONDS", + 0.05, + ) + caplog.set_level(logging.WARNING, logger="LiteLLM") + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="s1", name="stateful", url="http://upstream/mcp", transport=MCPTransport.http) + client: Final = _OfflineMCPClient(server_url=server.url, transport_type=MCPTransport.http) + gateway_session_id: Final = "gw-timeout" + key: Final = ( + gateway_session_id, + server.server_id, + await client.discovery_auth_fingerprint(), + client.session_settings(), + ) + session: Final = NeverClosingSession() + manager._upstream_sessions[key] = cast(PersistentMCPSession, session) + manager.track_gateway_session(gateway_session_id) + + await manager.close_all_upstream_sessions() + + assert session.closed + assert manager._upstream_sessions == {} + assert manager._live_gateway_sessions == frozenset() + assert "Timed out waiting for persistent MCP sessions to close during shutdown" in caplog.text + + +@pytest.mark.asyncio +async def test_upstream_session_is_not_reused_outside_an_mcp_protocol_request() -> None: + manager: Final = MCPServerManager() + server: Final = MCPServer(server_id="s1", name="stateful", url="http://upstream/mcp", transport=MCPTransport.http) + client: Final = _OfflineMCPClient(server_url=server.url, transport_type=MCPTransport.http) + context_token: Final = active_mcp_request_ctx_var.set(None) + + manager.track_gateway_session("gw-rest") + try: + session: Final = await manager._upstream_session_for( + client, server, {"mcp-session-id": "gw-rest"} + ) + assert session is None + finally: + manager.release_upstream_sessions("gw-rest") + active_mcp_request_ctx_var.reset(context_token) + + class TestSharedIdentifierPrefixWarning: """Two stored rows sharing lowercased alias-or-server_name publish one tool prefix; reload must surface them once so the ambiguity is visible.""" diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py index 4a214617142..4b46406ac69 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py @@ -11598,7 +11598,7 @@ async def test_passthrough_probes_bind_each_token_to_its_authorized_server(allow @pytest.mark.asyncio -async def test_streamable_http_admission_auth_sees_peeked_jsonrpc_body() -> None: +async def test_streamable_http_admission_auth_sees_only_peeked_jsonrpc_method() -> None: from litellm.constants import MCP_PEEKED_BODY_SCOPE_KEY from litellm.proxy._experimental.mcp_server import server as mcp_module @@ -11646,7 +11646,7 @@ async def test_streamable_http_admission_auth_sees_peeked_jsonrpc_body() -> None ): await mcp_module.handle_streamable_http_mcp(scope, receive, send) - assert auth_bodies == [_INITIALIZE] + assert auth_bodies == [b'{"method": "initialize"}'] assert downstream_bodies == [_INITIALIZE] stateless_handle.assert_not_awaited() send.assert_not_awaited() diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py index dda8f0b1044..7b9fc17840a 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_sigv4_auth.py @@ -7,16 +7,19 @@ tests for credential encryption, merge-on-update, and build_from_table. """ import json - -import pytest -from unittest.mock import patch, MagicMock, AsyncMock +from datetime import datetime, timedelta, timezone +from typing import Final +from unittest.mock import AsyncMock, MagicMock, patch import httpx2 - -from litellm.experimental_mcp_client.client import MCPSigV4Auth, MCPClient -from litellm.types.mcp import MCPAuth, MCPTransport +import pytest from prisma import models +from litellm.experimental_mcp_client.client import MCPClient, MCPSigV4Auth +from litellm.types.mcp import MCPAuth, MCPTransport + +_FUTURE_EXPIRATION: Final = datetime.now(timezone.utc) + timedelta(hours=1) + def _updated_row() -> models.LiteLLM_MCPServerTable: return models.LiteLLM_MCPServerTable.model_construct(server_id="test-server", transport="http", env={}, env_vars=[]) @@ -178,7 +181,7 @@ class TestMCPSigV4AssumeRole: "AccessKeyId": "ASSUMED_KEY", "SecretAccessKey": "ASSUMED_SECRET", "SessionToken": "ASSUMED_TOKEN", - "Expiration": "2026-03-30T12:00:00Z", + "Expiration": _FUTURE_EXPIRATION, } } @@ -197,6 +200,39 @@ class TestMCPSigV4AssumeRole: assert auth.credentials.secret_key == "ASSUMED_SECRET" assert auth.credentials.token == "ASSUMED_TOKEN" + def test_assume_role_signer_refreshes_expired_credentials(self): + """A retained signer re-assumes the role once the STS credentials expire.""" + mock_sts = MagicMock() + mock_sts.assume_role.side_effect = [ + { + "Credentials": { + "AccessKeyId": "ASSUMED_KEY_EXPIRED", + "SecretAccessKey": "ASSUMED_SECRET", + "SessionToken": "ASSUMED_TOKEN", + "Expiration": datetime.now(timezone.utc) - timedelta(minutes=1), + } + }, + { + "Credentials": { + "AccessKeyId": "ASSUMED_KEY_FRESH", + "SecretAccessKey": "ASSUMED_SECRET", + "SessionToken": "ASSUMED_TOKEN", + "Expiration": _FUTURE_EXPIRATION, + } + }, + ] + + with patch("boto3.client", return_value=mock_sts): + auth = MCPSigV4Auth( + aws_role_name="arn:aws:iam::123456789012:role/TestRole", + aws_region_name="us-east-1", + ) + request = httpx2.Request("POST", "https://example.com/mcp", content=b"{}") + signed = next(auth.auth_flow(request)) + + assert "Credential=ASSUMED_KEY_FRESH/" in signed.headers["Authorization"] + assert mock_sts.assume_role.call_count == 2 + def test_assume_role_with_explicit_source_credentials(self): """When aws_role_name + explicit keys are provided, keys are used as STS source identity.""" mock_sts = MagicMock() @@ -205,7 +241,7 @@ class TestMCPSigV4AssumeRole: "AccessKeyId": "ASSUMED_KEY", "SecretAccessKey": "ASSUMED_SECRET", "SessionToken": "ASSUMED_TOKEN", - "Expiration": "2026-03-30T12:00:00Z", + "Expiration": _FUTURE_EXPIRATION, } } @@ -233,7 +269,7 @@ class TestMCPSigV4AssumeRole: "AccessKeyId": "ASSUMED_KEY", "SecretAccessKey": "ASSUMED_SECRET", "SessionToken": "ASSUMED_TOKEN", - "Expiration": "2026-03-30T12:00:00Z", + "Expiration": _FUTURE_EXPIRATION, } } @@ -254,7 +290,7 @@ class TestMCPSigV4AssumeRole: "AccessKeyId": "AKIAIOSFODNN7EXAMPLE", "SecretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "SessionToken": "STS_SESSION_TOKEN", - "Expiration": "2026-03-30T12:00:00Z", + "Expiration": _FUTURE_EXPIRATION, } } @@ -285,7 +321,7 @@ class TestMCPSigV4AssumeRole: "AccessKeyId": "ASSUMED_KEY", "SecretAccessKey": "ASSUMED_SECRET", "SessionToken": "ASSUMED_TOKEN", - "Expiration": "2026-03-30T12:00:00Z", + "Expiration": _FUTURE_EXPIRATION, } } @@ -382,9 +418,6 @@ class TestMCPServerManagerSigV4: @pytest.mark.asyncio async def test_load_config_with_aws_sigv4(self, config_only_mcp_manager_factory): """Config loading correctly parses aws_sigv4 auth type and AWS fields.""" - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - MCPServerManager, - ) config = { "agentcore_tools": { @@ -497,7 +530,7 @@ class TestMCPServerManagerSigV4: "AccessKeyId": "ASSUMED_KEY", "SecretAccessKey": "ASSUMED_SECRET", "SessionToken": "ASSUMED_TOKEN", - "Expiration": "2026-03-30T12:00:00Z", + "Expiration": _FUTURE_EXPIRATION, } } @@ -1120,10 +1153,10 @@ class TestInheritCredentials: def test_inherits_sigv4_credentials(self): """SigV4 fields are copied from existing server to inherited credentials.""" + from litellm.proxy._types import NewMCPServerRequest from litellm.proxy.management_endpoints.mcp_management_endpoints import ( inherit_credentials_from_existing_server, ) - from litellm.proxy._types import NewMCPServerRequest from litellm.types.mcp_server.mcp_server_manager import MCPServer existing = MCPServer( diff --git a/tests/unit/proxy/common_utils/test_callback_config_validation.py b/tests/unit/proxy/common_utils/test_callback_config_validation.py index d10a96e2aef..a6544e56c57 100644 --- a/tests/unit/proxy/common_utils/test_callback_config_validation.py +++ b/tests/unit/proxy/common_utils/test_callback_config_validation.py @@ -1,7 +1,11 @@ +from collections.abc import Iterator +from typing import Final + import pytest from litellm.proxy.common_utils.callback_config_validation import ( callback_config_error, + conflicting_capture_error, conflicting_span_scope_error, cross_entry_family_error, logging_metadata_config_error, @@ -259,3 +263,114 @@ def test_arize_sampling_rates_are_not_family_credentials(): stored = [{"arize_api_key": "k1", "arize_success_sampling_rate": "0.5"}] assert cross_entry_family_error({"arize_success_sampling_rate": "0.1"}, stored) is None assert cross_entry_family_error({"arize_error_sampling_rate": "0.5"}, stored) is None + + +@pytest.fixture +def otel_v2_on(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + yield + is_otel_v2_enabled.cache_clear() + + +@pytest.mark.usefixtures("otel_v2_on") +@pytest.mark.parametrize("callback_name", ["langfuse_otel", "arize", "weave_otel", "newrelic"]) +@pytest.mark.parametrize("value", ["no_content", "span_only"]) +def test_capture_message_content_is_accepted_on_every_otel_v2_destination(callback_name: str, value: str) -> None: + assert callback_config_error(callback_name, {"capture_message_content": value}) is None + + +def test_capture_message_content_is_rejected_while_otel_v2_is_off(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + try: + error: Final = callback_config_error("langfuse_otel", {"capture_message_content": "no_content"}) + finally: + is_otel_v2_enabled.cache_clear() + assert error is not None and "capture_message_content" in error and "LITELLM_OTEL_V2" in error + + +@pytest.mark.parametrize("callback_name", ["langfuse", "datadog", "otel", "arize_phoenix", None]) +def test_capture_message_content_is_rejected_where_it_would_never_take_effect(callback_name: str | None) -> None: + error: Final = callback_config_error(callback_name, {"capture_message_content": "no_content"}) + assert error is not None and "capture_message_content" in error and "langfuse_otel" in error + + +@pytest.mark.usefixtures("otel_v2_on") +@pytest.mark.parametrize("value", ["full", "event_only", "span_and_event"]) +def test_an_unsupported_capture_message_content_is_rejected_on_key_logging_metadata(value: str) -> None: + error: Final = logging_metadata_config_error( + { + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {"capture_message_content": value}, + } + ] + } + ) + assert error is not None and "Invalid capture_message_content" in error + + +@pytest.mark.usefixtures("otel_v2_on") +def test_each_entry_keeps_its_own_capture_message_content() -> None: + error: Final = logging_metadata_config_error( + { + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {"capture_message_content": "span_only"}, + }, + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"capture_message_content": "no_content"}, + }, + ] + } + ) + assert error is None + + +@pytest.mark.usefixtures("otel_v2_on") +def test_a_failure_only_entry_rejects_capture_message_content() -> None: + error: Final = callback_config_error("langfuse_otel", {"capture_message_content": "no_content"}, "failure") + assert error is not None and "capture_message_content" in error and "success_and_failure" in error + + +@pytest.mark.parametrize( + ("stored", "rejected"), + [ + ([("langfuse_otel", {"capture_message_content": "no_content"})], True), + ([("langfuse_otel", {"capture_message_content": "span_only"})], False), + ([("langfuse_otel", {"langfuse_public_key": "pk"})], False), + ([("newrelic", {"capture_message_content": "no_content"})], False), + ], +) +def test_entries_for_one_backend_share_one_capture_message_content( + stored: list[tuple[str, dict[str, str]]], rejected: bool +) -> None: + """The backend's entries merge into one destination, so a second value would silently win or lose.""" + error: Final = conflicting_capture_error("langfuse_otel", {"capture_message_content": "span_only"}, stored) + assert (error is not None) is rejected + + +@pytest.mark.usefixtures("otel_v2_on") +def test_key_logging_entries_for_one_backend_may_not_disagree_on_capture_message_content() -> None: + def entry(callback_type: str, capture: str) -> dict[str, str | dict[str, str]]: + return { + "callback_name": "langfuse_otel", + "callback_type": callback_type, + "callback_vars": {"capture_message_content": capture}, + } + + error: Final = logging_metadata_config_error( + {"logging": [entry("success", "no_content"), entry("success_and_failure", "span_only")]} + ) + assert error is not None and "already set to 'no_content' by another langfuse_otel entry" in error diff --git a/tests/unit/proxy/common_utils/test_custom_openapi_spec.py b/tests/unit/proxy/common_utils/test_custom_openapi_spec.py index 485dbcef2f6..41ab613e7ea 100644 --- a/tests/unit/proxy/common_utils/test_custom_openapi_spec.py +++ b/tests/unit/proxy/common_utils/test_custom_openapi_spec.py @@ -4,11 +4,14 @@ Simple unit tests for CustomOpenAPISpec class. Tests basic functionality of OpenAPI schema generation. """ -from unittest.mock import Mock, patch +import json +from typing import Annotated, Literal +from unittest.mock import patch import pytest +from pydantic import BaseModel, ConfigDict, Field -from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec +from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec, inline_request_body class TestCustomOpenAPISpec: @@ -27,19 +30,13 @@ class TestCustomOpenAPISpec: }, } - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) - def test_add_chat_completion_request_schema( - self, mock_add_schema, base_openapi_schema - ): + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") + def test_add_chat_completion_request_schema(self, mock_add_schema, base_openapi_schema): """Test that chat completion schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema with patch("litellm.proxy._types.ProxyChatCompletionRequest") as mock_model: - result = CustomOpenAPISpec.add_chat_completion_request_schema( - base_openapi_schema - ) + result = CustomOpenAPISpec.add_chat_completion_request_schema(base_openapi_schema) mock_add_schema.assert_called_once_with( openapi_schema=base_openapi_schema, @@ -50,9 +47,7 @@ class TestCustomOpenAPISpec: ) assert result == base_openapi_schema - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") def test_add_embedding_request_schema(self, mock_add_schema, base_openapi_schema): """Test that embedding schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema @@ -69,19 +64,13 @@ class TestCustomOpenAPISpec: ) assert result == base_openapi_schema - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) - def test_add_responses_api_request_schema( - self, mock_add_schema, base_openapi_schema - ): + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") + def test_add_responses_api_request_schema(self, mock_add_schema, base_openapi_schema): """Test that responses API schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema with patch("litellm.types.llms.openai.ResponsesAPIRequestParams") as mock_model: - result = CustomOpenAPISpec.add_responses_api_request_schema( - base_openapi_schema - ) + result = CustomOpenAPISpec.add_responses_api_request_schema(base_openapi_schema) mock_add_schema.assert_called_once_with( openapi_schema=base_openapi_schema, @@ -123,15 +112,11 @@ def test_defs_rewritten_in_add_schema_to_components(): ) assert "$defs" not in openapi_schema assert ( - openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"][ - "items" - ]["anyOf"][0]["$ref"] + openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][0]["$ref"] == "#/components/schemas/UserMessage" ) assert ( - openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"][ - "items" - ]["anyOf"][1]["$ref"] + openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][1]["$ref"] == "#/components/schemas/AssistantMessage" ) @@ -188,14 +173,8 @@ def test_rewrite_defs_refs(): rewritten = CustomOpenAPISpec._rewrite_defs_refs(schema=schema, renames={}) assert "$defs" not in rewritten - assert ( - rewritten["properties"]["messages"]["items"]["anyOf"][0]["$ref"] - == "#/components/schemas/UserMessage" - ) - assert ( - rewritten["properties"]["messages"]["items"]["anyOf"][1]["$ref"] - == "#/components/schemas/AssistantMessage" - ) + assert rewritten["properties"]["messages"]["items"]["anyOf"][0]["$ref"] == "#/components/schemas/UserMessage" + assert rewritten["properties"]["messages"]["items"]["anyOf"][1]["$ref"] == "#/components/schemas/AssistantMessage" def test_get_pydantic_schema_generates_schema_for_responses_request_typed_dict(): @@ -390,3 +369,42 @@ def test_add_schema_to_components_renames_def_with_different_required_set(): assert schemas["Block"]["required"] == ["type", "x", "y"] assert schemas["Req_Block"]["required"] == ["keys", "type", "x", "y"] assert schemas["Req"]["properties"]["b"]["$ref"] == "#/components/schemas/Req_Block" + + +class _Cat(BaseModel): + kind: Literal["cat"] + lives: int = 9 + + model_config = ConfigDict(frozen=True) + + +class _Dog(BaseModel): + kind: Literal["dog"] + good: bool = True + + model_config = ConfigDict(frozen=True) + + +class _Shelter(BaseModel): + pets: dict[str, Annotated[_Cat | _Dog, Field(discriminator="kind")]] + favorite: _Cat | None = None + + model_config = ConfigDict(frozen=True) + + +def test_inline_request_body_expands_every_ref_so_the_operation_stands_alone(): + body = inline_request_body(_Shelter, {"pets": {"tom": {"kind": "cat"}}}) + + media = body["content"]["application/json"] + assert body["required"] is True + assert media["example"] == {"pets": {"tom": {"kind": "cat"}}} + assert "$ref" not in json.dumps(media["schema"]) + assert "$defs" not in media["schema"] + pet_schema = media["schema"]["properties"]["pets"]["additionalProperties"] + variants = {variant["properties"]["kind"]["const"]: variant for variant in pet_schema["oneOf"]} + assert variants["cat"]["properties"]["lives"]["default"] == 9 + assert variants["dog"]["properties"]["good"]["type"] == "boolean" + assert pet_schema["discriminator"] == {"propertyName": "kind"} + favorite = media["schema"]["properties"]["favorite"] + assert favorite["default"] is None + assert any(option.get("properties", {}).get("lives") for option in favorite["anyOf"]) diff --git a/tests/unit/proxy/common_utils/test_resource_ownership.py b/tests/unit/proxy/common_utils/test_resource_ownership.py new file mode 100644 index 00000000000..a13f66dd59c --- /dev/null +++ b/tests/unit/proxy/common_utils/test_resource_ownership.py @@ -0,0 +1,29 @@ +from typing import Final + +from litellm.litellm_core_utils import resource_ownership as core_resource_ownership +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.common_utils import resource_ownership as proxy_resource_ownership + +PREVIOUSLY_EXPOSED_NAMES: Final = ( + "Final", + "LitellmUserRoles", + "UserAPIKeyAuth", + "get_primary_resource_owner_scope", + "get_resource_owner_scopes", + "is_proxy_admin", + "user_can_access_resource_owner", +) + + +def test_proxy_path_reexports_core_objects() -> None: + mismatched: Final = [ + name + for name in PREVIOUSLY_EXPOSED_NAMES + if getattr(proxy_resource_ownership, name) is not getattr(core_resource_ownership, name) + ] + assert mismatched == [] + + +def test_moved_module_uses_the_proxy_types_objects() -> None: + assert core_resource_ownership.LitellmUserRoles is LitellmUserRoles + assert core_resource_ownership.UserAPIKeyAuth is UserAPIKeyAuth diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index ed78fc7063a..87b2b64ad8d 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -12,17 +12,19 @@ import respx from fastapi import FastAPI from fastapi.routing import APIRoute from fastapi.testclient import TestClient +from pydantic import TypeAdapter from starlette.routing import Match import litellm from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features -from litellm.proxy.decisions_endpoints.endpoints import decisions, systemone +from litellm.proxy.decisions_endpoints.endpoints import decisions, router, systemone from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder from litellm.proxy.proxy_server import ( app, cleanup_router_config_variables, initialize, ) +from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody _INPUT_TOKENS: Final[int] = 367 _OUTPUT_TOKENS: Final[int] = 3 @@ -688,3 +690,63 @@ def test_with_lazy_routes_disabled_a_config_pass_through_at_v1_decisions_still_w assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"} assert _serving_endpoint(bare, "/v1/decisions") is pass_through assert _serving_endpoint(bare, "/decisions") is decisions + + +@pytest.fixture(scope="module") +def decisions_openapi() -> dict[str, object]: + app = FastAPI() + app.include_router(router) + return app.openapi() + + +@pytest.mark.parametrize( + ("path", "body_adapter", "response_schema", "required"), + ( + ("/v1/systemone", TypeAdapter(DecisionsRequestBody), "DecisionsResponse", {"model", "state", "questions"}), + ("/systemone", TypeAdapter(DecisionsRequestBody), "DecisionsResponse", {"model", "state", "questions"}), + ( + "/v1/decisions", + TypeAdapter(OpenAIDecisionRequestBody), + "OpenAIDecisionResponse", + {"model", "input", "questions"}, + ), + ( + "/decisions", + TypeAdapter(OpenAIDecisionRequestBody), + "OpenAIDecisionResponse", + {"model", "input", "questions"}, + ), + ), +) +def test_swagger_documents_the_request_format_and_response_of_each_decisions_route( + decisions_openapi: dict[str, object], + path: str, + body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody], + response_schema: str, + required: set[str], +) -> None: + operation = decisions_openapi["paths"][path]["post"] + media = operation["requestBody"]["content"]["application/json"] + + assert "Decisions" in operation["summary"] or "System One" in operation["summary"] + assert "docs.litellm.ai/docs/decisions" in operation["description"] + assert required <= set(media["schema"]["required"]) + assert required <= set(media["schema"]["properties"]) + assert "$ref" not in json.dumps(media["schema"]) + assert media["schema"]["properties"]["model"]["description"] + + example = {key: value for key, value in media["example"].items() if key != "model"} + assert len(body_adapter.validate_python(example).questions) == 3 + + response_ref = operation["responses"]["200"]["content"]["application/json"]["schema"]["$ref"] + assert response_ref == f"#/components/schemas/{response_schema}" + assert "answers" in decisions_openapi["components"]["schemas"][response_schema]["properties"] + + +def test_the_two_decisions_routes_document_different_request_formats(decisions_openapi: dict[str, object]) -> None: + def properties(path: str) -> set[str]: + operation = decisions_openapi["paths"][path]["post"] + return set(operation["requestBody"]["content"]["application/json"]["schema"]["properties"]) + + assert "state" in properties("/v1/systemone") and "state" not in properties("/v1/decisions") + assert "input" in properties("/v1/decisions") and "input" not in properties("/v1/systemone") diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 9905ee619c2..ee39df92ef2 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -7817,7 +7817,7 @@ async def test_process_single_key_update(): mock_delete_cache.return_value = None # Mock hash_token (imported from litellm.proxy._types) - with patch("litellm.proxy._types.hash_token") as mock_hash: + with patch("litellm.types.proxy.auth.user_api_key_auth.hash_token") as mock_hash: mock_hash.return_value = "hashed-test-key-123" # Mock _hash_token_if_needed @@ -7963,7 +7963,7 @@ async def test_bulk_update_keys_success(monkeypatch): with patch( "litellm.proxy.management_endpoints.key_management_endpoints.delete_cache_key_object" ): - with patch("litellm.proxy._types.hash_token") as mock_hash: + with patch("litellm.types.proxy.auth.user_api_key_auth.hash_token") as mock_hash: mock_hash.side_effect = ["hashed-key-1", "hashed-key-2"] def _hash_for_bulk_success(token: str) -> str: @@ -8091,7 +8091,7 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): with patch( "litellm.proxy.management_endpoints.key_management_endpoints.delete_cache_key_object" ): - with patch("litellm.proxy._types.hash_token") as mock_hash: + with patch("litellm.types.proxy.auth.user_api_key_auth.hash_token") as mock_hash: mock_hash.return_value = "hashed-key-1" def _hash_for_bulk_partial(token: str) -> str: @@ -10733,6 +10733,7 @@ class TestValidateKeyAliasFormat: "foo\n- !grant\n role: !!admin\n member: attacker", "foo\rbar", "foo\x00bar", + "my-test-key\t", ], ) def test_validate_key_alias_format_rejects_traversal_and_control_chars_even_when_flag_disabled( @@ -10751,6 +10752,8 @@ class TestValidateKeyAliasFormat: _validate_key_alias_format(unsafe_alias) assert str(exc.value.code) == "400" assert "Invalid key_alias" in str(exc.value.message) + assert "control characters" in str(exc.value.message) + assert '".." path segment' in str(exc.value.message) def test_validate_key_alias_format_valid(self, monkeypatch): from litellm.proxy.management_endpoints.key_management_endpoints import ( diff --git a/tests/unit/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_callback_endpoints.py index f576bac4742..ad5b6903ece 100644 --- a/tests/unit/proxy/management_endpoints/test_team_callback_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_callback_endpoints.py @@ -7,6 +7,7 @@ redacted audit rows for callback mutations. """ import json +from collections.abc import Iterator from typing import Final from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -1712,3 +1713,62 @@ def test_add_team_callback_accepts_arize_sampling_rate_vars(): ) assert data.callback_vars["arize_success_sampling_rate"] == "0.5" assert data.callback_vars["arize_error_sampling_rate"] == "1.0" + + +@pytest.mark.asyncio +async def test_add_team_callbacks_rejects_capture_message_content_on_a_non_otel_v2_callback(patched_prisma): + data: Final = AddTeamCallback( + callback_name="langfuse", + callback_type="success", + callback_vars={ + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "capture_message_content": "no_content", + }, + ) + with pytest.raises(HTTPException) as exc: + await add_team_callbacks( + data=data, + http_request=Mock(spec=Request), + team_id="team-victim", + user_api_key_dict=_admin_auth(), + ) + assert exc.value.status_code == 400 + assert "capture_message_content" in str(exc.value.detail) + patched_prisma.db.litellm_teamtable.update.assert_not_called() + + +@pytest.fixture +def otel_v2_on(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + yield + is_otel_v2_enabled.cache_clear() + + +@pytest.mark.usefixtures("otel_v2_on") +@pytest.mark.asyncio +@pytest.mark.parametrize("capture", ["no_content", "span_only"]) +async def test_add_team_callbacks_stores_capture_message_content_in_the_existing_callback_metadata( + patched_prisma, capture: str +) -> None: + data: Final = AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={ + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "capture_message_content": capture, + }, + ) + await add_team_callbacks( + data=data, + http_request=Mock(spec=Request), + team_id="team-victim", + user_api_key_dict=_admin_auth(), + ) + patched_prisma.db.litellm_teamtable.update.assert_awaited_once() + stored: Final = json.loads(patched_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"]) + assert [entry["callback_vars"].get("capture_message_content") for entry in stored["logging"]] == [capture] diff --git a/tests/unit/test_check_openapi_docs.py b/tests/unit/test_check_openapi_docs.py new file mode 100644 index 00000000000..d7122541416 --- /dev/null +++ b/tests/unit/test_check_openapi_docs.py @@ -0,0 +1,54 @@ +"""Tests for scripts/check_openapi_docs.py, the scan behind scripts/openapi_docs_gate.py.""" + +from typing import Final + +import check_openapi_docs as checker + +_BODY: Final = {"content": {"application/json": {"schema": {"type": "object"}}}} +_RESPONSES: Final = {"200": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Out"}}}}} +_DOCUMENTED: Final = {"description": "Does a thing", "requestBody": _BODY, "responses": _RESPONSES} + + +def test_a_fully_documented_post_has_no_findings() -> None: + assert checker.violations({"paths": {"/v1/thing": {"post": _DOCUMENTED}}}) == () + + +def test_each_missing_piece_is_its_own_rule() -> None: + bare: Final = {"summary": "Thing", "responses": {"200": {"description": "Successful Response"}}} + codes: Final = {v.code for v in checker.violations({"paths": {"/v1/thing": {"post": bare}}})} + assert codes == {"OA001", "OA002", "OA003"} + + +def test_only_the_missing_piece_is_reported() -> None: + no_body: Final = {**_DOCUMENTED, "requestBody": {}} + blank_description: Final = {**_DOCUMENTED, "description": " "} + untyped_200: Final = {**_DOCUMENTED, "responses": {"200": {"content": {"application/json": {"schema": {}}}}}} + spec: Final = { + "paths": { + "/a": {"post": no_body}, + "/b": {"post": blank_description}, + "/c": {"post": untyped_200}, + } + } + assert checker.violations(spec) == ( + checker.Violation("POST", "/a", "OA001"), + checker.Violation("POST", "/b", "OA002"), + checker.Violation("POST", "/c", "OA003"), + ) + + +def test_non_post_operations_are_not_gated() -> None: + spec: Final = {"paths": {"/v1/thing": {"get": {}, "delete": {}, "put": {}, "post": _DOCUMENTED}}} + assert checker.violations(spec) == () + + +def test_findings_are_sorted_by_path_so_counts_are_stable() -> None: + spec: Final = {"paths": {"/z": {"post": {}}, "/a": {"post": {}}}} + assert [v.path for v in checker.violations(spec)] == ["/a", "/a", "/a", "/z", "/z", "/z"] + + +def test_main_reads_a_spec_file_and_prints_one_line_per_finding(tmp_path, capsys) -> None: + spec_file: Final = tmp_path / "openapi.json" + spec_file.write_text('{"paths": {"/v1/thing": {"post": {"description": "x", "requestBody": {}}}}}') + assert checker.main([str(spec_file)]) == 0 + assert capsys.readouterr().out.splitlines() == ["POST /v1/thing OA001", "POST /v1/thing OA003"] diff --git a/tests/unit/test_openapi_docs_gate.py b/tests/unit/test_openapi_docs_gate.py new file mode 100644 index 00000000000..fa8a5b1c7e3 --- /dev/null +++ b/tests/unit/test_openapi_docs_gate.py @@ -0,0 +1,60 @@ +"""Tests for scripts/openapi_docs_gate.py. + +The blame rule lives in scripts/lint_base_counts.py and is tested there; what is tested +here is the checker output parse and `introduced`, the diff scan that names the new +endpoint behind a breach. +""" + +from pathlib import Path +from typing import Final + +import openapi_docs_gate as gate + + +def test_editing_the_checker_rekeys_the_base_counts(tmp_path: Path) -> None: + checker: Final = tmp_path / "check.py" + checker.write_text("print('v1')\n") + before: Final = gate.checker_identity(checker).artifact_name("abc123") + checker.write_text("print('v2')\n") + assert gate.checker_identity(checker).artifact_name("abc123") != before + + +def test_parse_violations_skips_anything_that_is_not_a_finding_line() -> None: + out: Final = "Some warning from an import\nPOST /v1/decisions OA001\nPOST /v1/decisions OA003\nnot a line\n" + assert gate.parse_violations(out) == ( + gate.Violation("POST", "/v1/decisions", "OA001"), + gate.Violation("POST", "/v1/decisions", "OA003"), + ) + + +def test_count_by_rule_sums_findings_per_code() -> None: + violations: Final = ( + gate.Violation("POST", "/a", "OA001"), + gate.Violation("POST", "/b", "OA001"), + gate.Violation("POST", "/b", "OA003"), + ) + assert dict(gate.count_by_rule(violations)) == {"OA001": 2, "OA003": 1} + + +def test_introduced_keeps_only_routes_registered_on_added_lines() -> None: + diff: Final = ( + "diff --git a/litellm/proxy/x.py b/litellm/proxy/x.py\n" + "--- a/litellm/proxy/x.py\n" + "+++ b/litellm/proxy/x.py\n" + "@@ -0,0 +3,2 @@\n" + '+@router.post("/v1/new")\n' + "+async def new(request: Request): ...\n" + "@@ -10,1 +20,1 @@\n" + '-@router.post("/v1/removed")\n' + ) + violations: Final = ( + gate.Violation("POST", "/v1/new", "OA001"), + gate.Violation("POST", "/v1/old", "OA001"), + gate.Violation("POST", "/v1/removed", "OA001"), + ) + assert gate.introduced(violations, gate.added_text(diff)) == (gate.Violation("POST", "/v1/new", "OA001"),) + + +def test_introduced_does_not_match_a_path_that_is_only_a_prefix_of_an_added_one() -> None: + added: Final = gate.added_text('+@router.post("/v1/decisions/batch")\n') + assert gate.introduced((gate.Violation("POST", "/v1/decisions", "OA002"),), added) == () diff --git a/tests/unit/test_scaledown_model_prices.py b/tests/unit/test_scaledown_model_prices.py new file mode 100644 index 00000000000..02c7983f753 --- /dev/null +++ b/tests/unit/test_scaledown_model_prices.py @@ -0,0 +1,46 @@ +from typing import Final + +import pytest + +import litellm +from litellm.types.utils import Usage + +SCALEDOWN_MODELS: Final = ( + "scaledown/classify", + "scaledown/decisions", + "scaledown/extract", + "scaledown/summarize", + "scaledown/compress", +) + +# Source: ScaleDown's standard rate of $0.05 per million input tokens with output unbilled, +# as shown in the ScaleDown usage dashboard (checked 2026-10-09) and its billing service. +INPUT_COST_PER_MILLION_TOKENS: Final = 0.05 + + +@pytest.mark.parametrize("model", SCALEDOWN_MODELS) +def test_scaledown_model_is_priced_on_input_tokens_only(model: str) -> None: + entry: Final = litellm.model_cost[model] + + assert entry["litellm_provider"] == "scaledown" + assert entry["mode"] == "chat" + assert entry["input_cost_per_token"] * 1_000_000 == pytest.approx(INPUT_COST_PER_MILLION_TOKENS) + assert entry["output_cost_per_token"] == 0.0 + + +@pytest.mark.parametrize("model", SCALEDOWN_MODELS) +def test_scaledown_cost_ignores_output_tokens(model: str) -> None: + prompt_cost, completion_cost = litellm.cost_per_token( + model=model, + custom_llm_provider="scaledown", + usage_object=Usage(prompt_tokens=1_000_000, completion_tokens=500_000), + ) + + assert prompt_cost == pytest.approx(INPUT_COST_PER_MILLION_TOKENS) + assert completion_cost == 0.0 + + +def test_every_scaledown_entry_in_the_cost_map_is_covered_here() -> None: + in_map: Final = {key for key in litellm.model_cost if key.startswith("scaledown/")} + + assert in_map == set(SCALEDOWN_MODELS) diff --git a/tests/unit/types/test_utils.py b/tests/unit/types/test_utils.py new file mode 100644 index 00000000000..09063d894e7 --- /dev/null +++ b/tests/unit/types/test_utils.py @@ -0,0 +1,40 @@ +import pytest + +from litellm.types.utils import CAPTURE_MESSAGE_CONTENT_VALUES, CaptureMessageContent, captures_span_content + + +@pytest.mark.parametrize( + "mode", + [ + CaptureMessageContent.NO_CONTENT, + CaptureMessageContent.SPAN_ONLY, + CaptureMessageContent.EVENT_ONLY, + CaptureMessageContent.SPAN_AND_EVENT, + ], +) +def test_capture_message_content_members_are_plain_strings(mode: str) -> None: + assert type(mode) is str + assert (str(mode), f"{mode}", repr(mode)) == (mode, mode, repr(str(mode))) + + +def test_a_destination_may_only_set_the_span_capture_modes() -> None: + assert CAPTURE_MESSAGE_CONTENT_VALUES == {"no_content", "span_only"} + + +def test_capture_message_content_constructor_accepts_any_string() -> None: + assert CaptureMessageContent("custom") == "custom" + + +@pytest.mark.parametrize( + ("mode", "captures"), + [ + ("no_content", False), + ("span_only", True), + ("event_only", False), + ("span_and_event", True), + ("bogus", False), + (None, False), + ], +) +def test_captures_span_content(mode: str | None, captures: bool) -> None: + assert captures_span_content(mode) is captures diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneForm.tsx index 5d53bc2d620..cfffb30bb5d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneForm.tsx @@ -27,7 +27,12 @@ import { import type { SystemOnePayloadValidation } from "./lib/validatePayload"; const QUESTION_TYPES: readonly QuestionType[] = ["choice", "noul", "score"]; -const TYPE_LABELS: Record = { choice: "Choice", noul: "Yes / no", score: "Score" }; +const TYPE_LABELS: Record = { choice: "Choice", noul: "noul", score: "Score" }; +const TYPE_HELP: Record = { + choice: undefined, + noul: "noul is a yes / no question", + score: undefined, +}; const isQuestionType = (value: unknown): value is QuestionType => QUESTION_TYPES.some((type) => type === value); @@ -219,7 +224,7 @@ function QuestionCard({ name, question, taken, onChange, onRename, onRemove }: Q {QUESTION_TYPES.map((type) => ( - + {TYPE_LABELS[type]} ))} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx index ab0a67dbbab..eb75c836b2b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx @@ -11,7 +11,7 @@ import { import type { ReactElement } from "react"; import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import SystemOneUI from "./SystemOneUI"; +import SystemOneUI, { NOUL_HELP } from "./SystemOneUI"; import type { SystemOneResponse } from "./lib/schemas"; import type { OpenAIDecisionsResponse } from "./lib/openAIDecisions"; @@ -71,6 +71,12 @@ const selectEndpoint = async (endpoint: "/v1/decisions" | "/v1/systemone") => { await userEvent.click(await screen.findByRole("option", { name: endpoint, exact: true })); }; +const selectPreset = async (label: string) => { + screen.getByRole("combobox", { name: "Example" }).focus(); + await userEvent.keyboard("{ArrowDown}"); + await userEvent.click(await screen.findByRole("option", { name: label })); +}; + const createResponse = (body: SystemOneResponse | OpenAIDecisionsResponse, status = 200, errorText = "") => ({ ok: status >= 200 && status < 300, @@ -99,20 +105,81 @@ describe("SystemOneUI integration", () => { document.cookie = "token=; path=/; max-age=0"; }); - it("restores the example after the request is edited", async () => { - const user = userEvent.setup(); + it("switches between example presets and reads Custom once the request is edited", async () => { render(); const editor = await openJsonEditor(); const example = (editor as HTMLTextAreaElement).value; - const resetButton = screen.getByRole("button", { name: "Reset example" }); - expect(resetButton).toBeDisabled(); + const picker = screen.getByRole("combobox", { name: "Example" }); + expect(picker).toHaveTextContent("Triage a bug report"); fireEvent.change(editor, { target: { value: "{}" } }); - expect(resetButton).toBeEnabled(); - await user.click(resetButton); - + expect(picker).toHaveTextContent("Custom"); + await selectPreset("Triage a bug report"); expect(editor).toHaveValue(example); - expect(resetButton).toBeDisabled(); + expect(picker).toHaveTextContent("Triage a bug report"); + + await selectPreset("Score a sales lead"); + expect(picker).toHaveTextContent("Score a sales lead"); + expect(JSON.parse((editor as HTMLTextAreaElement).value)).toMatchObject({ questions: { fit: { type: "score" } } }); + }); + + it("clears the whole request and lets a preset be picked from nothing", async () => { + const user = userEvent.setup(); + render(); + const clear = screen.getByRole("button", { name: "Clear" }); + expect(screen.getByRole("textbox", { name: "Input" })).toHaveDisplayValue(/streaming responses/); + + await user.click(clear); + expect(screen.getByRole("textbox", { name: "Input" })).toHaveValue(""); + expect(screen.queryAllByRole("group", { name: /Question / })).toHaveLength(0); + expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); + expect(clear).toBeDisabled(); + expect(screen.getByRole("combobox", { name: "Example" })).toHaveTextContent("Custom"); + + await selectPreset("Route a support ticket"); + expect(screen.getByRole("textbox", { name: "Input" })).toHaveDisplayValue(/charged twice/); + expect(screen.getAllByRole("group", { name: /Question / })).toHaveLength(1); + expect(screen.getByRole("button", { name: "Send" })).toBeEnabled(); + }); + + it("keeps the preset label after picking a model and leaves an omitted model out when clearing", async () => { + mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS)); + render(); + await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest")); + const picker = screen.getByRole("combobox", { name: "Example" }); + const editor = await openJsonEditor(); + const example = JSON.parse((editor as HTMLTextAreaElement).value) as Record; + const { model: _model, ...withoutModel } = example; + + fireEvent.change(editor, { + target: { value: JSON.stringify({ ...withoutModel, model: "pplx-decider" }, null, 2) }, + }); + expect(picker).toHaveTextContent("Triage a bug report"); + + fireEvent.change(editor, { target: { value: JSON.stringify(withoutModel, null, 2) } }); + expect(picker).toHaveTextContent("Triage a bug report"); + await userEvent.click(screen.getByRole("button", { name: "Clear" })); + expect(JSON.parse((editor as HTMLTextAreaElement).value)).toEqual({ state: "", questions: {} }); + await selectPreset("Score a sales lead"); + expect(JSON.parse((editor as HTMLTextAreaElement).value)).not.toHaveProperty("model"); + }); + + it("offers the same presets on /v1/decisions in the OpenAI shape and clears to an empty input", async () => { + const user = userEvent.setup(); + render(); + await selectEndpoint("/v1/decisions"); + await selectPreset("Moderate a forum comment"); + await user.click(screen.getByRole("tab", { name: "JSON" })); + const editor = screen.getByRole("textbox", { name: "Decisions JSON payload" }) as HTMLTextAreaElement; + const payload = JSON.parse(editor.value); + expect(payload.input).toMatch(/stream=false/); + expect(payload.questions).toEqual([ + { type: "predicate", name: "breaks_rules", instructions: "Does this comment break the community guidelines?" }, + ]); + + await user.click(screen.getByRole("button", { name: "Clear" })); + expect(JSON.parse(editor.value)).toEqual({ input: "", questions: [] }); + expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); }); it("offers both literal proxy endpoints without TypeSafe or product prefixes", async () => { @@ -227,7 +294,7 @@ describe("SystemOneUI integration", () => { await selectEndpoint("/v1/decisions"); expect(screen.getByRole("textbox", { name: "Decisions JSON payload" })).toHaveValue(JSON.stringify(payload)); - await user.click(screen.getByRole("button", { name: "Reset example" })); + await selectPreset("Triage a bug report"); expect(screen.getByRole("textbox", { name: "Decisions JSON payload" })).toHaveValue(nativeExample); await selectEndpoint("/v1/systemone"); expect(screen.getByRole("textbox", { name: "System One JSON payload" })).toHaveValue(systemOneDraft); @@ -403,7 +470,7 @@ describe("SystemOneUI integration", () => { ); expect(next).toEqual({ ...payload, questions: [{ ...payload.questions[0], instructions: "Is it urgent?" }] }); await user.click(screen.getByRole("tab", { name: "Form" })); - await user.click(screen.getByRole("button", { name: "Reset example" })); + await selectPreset("Triage a bug report"); expect(screen.getByRole("textbox", { name: "Input" })).toBeInTheDocument(); expect(screen.getAllByRole("group", { name: /Question \d/ })).toHaveLength(3); await selectEndpoint("/v1/systemone"); @@ -853,6 +920,22 @@ describe("SystemOneUI integration", () => { fireEvent.blur(field); } + it("explains noul on hover in the banner and on the answer type option", async () => { + const user = userEvent.setup(); + render(); + + await user.hover(screen.getByText("noul", { selector: "span.cursor-help" })); + expect(await screen.findByText(NOUL_HELP)).toBeInTheDocument(); + + question("has_repro_steps").getByRole("combobox", { name: "Answer type" }).focus(); + await user.keyboard("{ArrowDown}"); + expect(await screen.findByRole("option", { name: "noul" })).toHaveAttribute( + "title", + "noul is a yes / no question", + ); + expect(screen.getByRole("option", { name: "Choice" })).not.toHaveAttribute("title"); + }); + it("sends a choice and a yes/no question built in the form without opening JSON", async () => { const user = userEvent.setup(); render(); @@ -874,7 +957,7 @@ describe("SystemOneUI integration", () => { fill(question("team").getByRole("textbox", { name: "Description of infra" }), "Deploys"); await user.click(screen.getByRole("button", { name: "Add question" })); - await pickAnswerType(user, "question_1", "Yes / no"); + await pickAnswerType(user, "question_1", "noul"); rename(question("question_1").getByRole("textbox", { name: "Question name" }), "urgent"); fill(question("urgent").getByRole("textbox", { name: "Instructions" }), "Is this blocking users?"); fill(question("urgent").getByRole("textbox", { name: "Yes means" }), "Nobody can log in"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx index 02d42851b8e..3d2d4c0f1a0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx @@ -5,14 +5,15 @@ import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { SimpleTooltip } from "@/components/ui/tooltip"; import { SearchSelect } from "@/components/shared/SearchSelect"; import { DECISIONS_DOCS_URL } from "@/lib/decisionModels"; import { uiHref } from "@/utils/uiHref"; import { useMutation } from "@tanstack/react-query"; -import { Code, Info, LoaderCircle, RotateCcw, Send } from "lucide-react"; +import { Code, Eraser, Info, LoaderCircle, Send } from "lucide-react"; import { useEffect, useMemo, useRef, useState } from "react"; import { makePlaygroundDecisionRequest, type PlaygroundDecisionRequest } from "../../llm_calls/system_one"; -import { decisionsExample, openAIDecisionsExample } from "./lib/example"; +import { DECISION_PRESETS, emptyPayload, matchPreset, presetPayload, type DecisionPreset } from "./lib/example"; import { payloadModel, withPayloadModel } from "./lib/payloadModel"; import JsonEditor from "./JsonEditor"; import QuestionBreakdown from "./QuestionBreakdown"; @@ -35,8 +36,37 @@ interface SystemOneSendVariables { signal: AbortSignal; } +export const NOUL_HELP = "noul is a yes / no question. The model returns the probability that the answer is yes."; +const CUSTOM_PRESET = "custom"; const DECISION_MODELS_DISCUSSION_URL = "https://github.com/BerriAI/litellm/discussions/44231"; +interface PresetPickerProps { + activePreset: DecisionPreset | undefined; + onPick: (id: string | null) => void; +} + +function PresetPicker({ activePreset, onPick }: PresetPickerProps) { + return ( + + ); +} + function getCustomProxyBaseUrl(): string | undefined { return typeof window === "undefined" ? undefined : window.sessionStorage.getItem("customProxyBaseUrl") || undefined; } @@ -53,9 +83,10 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = const [endpoint, setEndpoint] = useState("/v1/systemone"); const [drafts, setDrafts] = useState>>({}); const [view, setView] = useState("form"); - const example = endpoint === "/v1/decisions" ? openAIDecisionsExample : decisionsExample; - const examplePayload = JSON.stringify(example(decisionModels[0]), null, 2); - const rawPayload = drafts[endpoint] ?? examplePayload; + const rawPayload = drafts[endpoint] ?? presetPayload(DECISION_PRESETS[0], endpoint, decisionModels[0]); + const draftModel = drafts[endpoint] === undefined ? decisionModels[0] : payloadModel(rawPayload); + const activePreset = matchPreset(endpoint, rawPayload); + const clearedPayload = emptyPayload(endpoint, draftModel); const activeController = useRef(null); const validated = useMemo( () => @@ -95,9 +126,11 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = } } - function handleResetExample() { - clearRequestState(); - setDrafts((current) => Object.fromEntries(Object.entries(current).filter(([key]) => key !== endpoint))); + function handlePresetPick(id: string | null) { + const preset = DECISION_PRESETS.find((candidate) => candidate.id === id); + if (preset !== undefined) { + handlePayloadChange(presetPayload(preset, endpoint, draftModel)); + } } function handleModelPick(model: string | null) { @@ -204,9 +237,14 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = )}
- {view === "json" && (