From 2bf0cddcc1523740a47b35fd1d566e7249d1ca04 Mon Sep 17 00:00:00 2001 From: shrey-berri Date: Wed, 30 Sep 2026 13:04:49 -0700 Subject: [PATCH 1/7] fix(bedrock): keep applicable beta headers (#43829) --- litellm/anthropic_beta_headers_config.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 0e0eeff83fe..7bb4c6e58df 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -34,7 +34,9 @@ "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19", "web-fetch-2025-09-10": "web-fetch-2025-09-10", "web-search-2025-03-05": "web-search-2025-03-05", - "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01" + "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01", + "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", + "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01" }, "azure_ai": { "advisor-tool-2026-03-01": null, @@ -136,7 +138,9 @@ "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19", "web-fetch-2025-09-10": null, "web-search-2025-03-05": null, - "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01" + "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01", + "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", + "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01" }, "bedrock_mantle": { "advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19", From 405ed414cb155a75eef1118119f4d9d7a7ab762d Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Wed, 30 Sep 2026 13:05:22 -0700 Subject: [PATCH 2/7] feat(agents): authenticate Entra identities and delegated requests (#43722) * feat(agents): authenticate Entra identities and delegated requests * fix(agents): enforce target policy and preserve trusted authentication * fix(proxy): make inference model selection exhaustive * fix(agents): authorize targets against current database policy * fix(proxy): make exhaustive model resolution return explicit --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .../mcp_server/idp_token_exchange.py | 5 + .../mcp_server/rest_endpoints.py | 33 +- litellm/proxy/_types.py | 2 + .../proxy/agent_endpoints/a2a_endpoints.py | 7 + litellm/proxy/agent_endpoints/a2a_routing.py | 2 +- .../auth/agent_permission_handler.py | 48 +- .../auth/managed_authorization.py | 172 +++++- litellm/proxy/auth/handle_jwt.py | 189 ++++-- litellm/proxy/auth/user_api_key_auth.py | 58 +- litellm/proxy/common_request_processing.py | 12 +- .../proxy/common_utils/http_parsing_utils.py | 38 +- .../common_utils/registry_read_through.py | 5 +- .../hooks/parallel_request_limiter_v3.py | 21 +- .../proxy/hooks/proxy_track_cost_callback.py | 10 +- litellm/proxy/image_endpoints/endpoints.py | 18 +- litellm/proxy/litellm_pre_call_utils.py | 14 +- .../sso/agent_subject_enrollment.py | 49 ++ litellm/proxy/management_endpoints/ui_sso.py | 27 + litellm/proxy/proxy_server.py | 17 +- .../spend_management_endpoints.py | 32 +- .../spend_tracking/spend_tracking_utils.py | 1 + .../mcp_server/test_idp_token_exchange.py | 11 + .../mcp_server/test_mcp_server.py | 6 +- .../mcp_server/test_mcp_stale_session.py | 41 +- .../mcp_server/test_rest_endpoints.py | 26 + .../auth/test_agent_permission_handler.py | 116 ++++ .../auth/test_managed_authorization.py | 296 ++++++++- .../agent_endpoints/test_a2a_endpoints.py | 7 +- .../proxy/agent_endpoints/test_endpoints.py | 3 + .../proxy/auth/test_handle_jwt.py | 584 ++++++++++++++---- .../proxy/auth/test_user_api_key_auth.py | 278 +++++++++ .../common_utils/test_http_parsing_utils.py | 42 +- .../test_registry_read_through.py | 34 +- .../hooks/test_parallel_request_limiter_v3.py | 114 ++++ .../hooks/test_proxy_track_cost_callback.py | 31 + .../sso/test_agent_subject_enrollment.py | 114 ++++ .../proxy/management_endpoints/test_ui_sso.py | 52 ++ .../proxy/proxy_server/test_proxy_config.py | 40 +- .../test_spend_management_endpoints.py | 44 +- .../test_spend_query_optimization.py | 10 +- .../test_spend_tracking_utils.py | 36 ++ .../proxy/test_route_a2a_models.py | 3 + 42 files changed, 2383 insertions(+), 265 deletions(-) create mode 100644 litellm/proxy/management_endpoints/sso/agent_subject_enrollment.py create mode 100644 tests/test_litellm/proxy/management_endpoints/sso/test_agent_subject_enrollment.py diff --git a/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py b/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py index a437df17e6a..15632eb4783 100644 --- a/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py +++ b/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py @@ -170,6 +170,11 @@ async def identity_from_subject_token( return _refusal_for(denied, denied.message) except Exception as denied: # noqa: BLE001 # auth_jwt raises a plain Exception on signature and claim failures return _refusal_for(denied, denied) + if result.get("agent_id") is not None: + return SubjectTokenRefusal( + error="invalid_request", + description="Agent tokens require direct JWT authentication; this exchange supports users only", + ) user_id: Final = result["user_id"] if user_id is None: return SubjectTokenRefusal(error="invalid_request", description="subject_token names no user the gateway knows") diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index c7ee4cdb3c0..12cdab59e0f 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from datetime import datetime from traceback import walk_tb from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal +from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict from uuid import uuid4 import anyio @@ -14,6 +14,7 @@ import httpx2 from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from pydantic import ValidationError from starlette.datastructures import Headers +from typing_extensions import ReadOnly from litellm._logging import verbose_logger from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_TOOL_LISTING_TIMEOUT @@ -63,7 +64,27 @@ if TYPE_CHECKING: from litellm.proxy.utils import ProxyLogging from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers from litellm.types.mcp import MCPAuth -from litellm.types.utils import CallTypes +from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall + + +class _MCPModelMetadata(TypedDict): + model_group: ReadOnly[str] + + +def _stamp_mcp_tool_metadata(logging_obj: "LiteLLMLoggingObj | None", server_id: str, tool_name: str) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + + if logging_obj is None: + return + server: Final = global_mcp_server_manager.get_mcp_server_by_id( + server_id + ) or global_mcp_server_manager.get_mcp_server_by_name(server_id) + metadata: Final[StandardLoggingMCPToolCall] = { + "name": tool_name, + "mcp_server_name": server.name if server is not None else server_id, + } + logging_obj.model_call_details["mcp_tool_call_metadata"] = metadata + MCP_AVAILABLE: bool = True try: @@ -1193,6 +1214,12 @@ if MCP_AVAILABLE: }, ) + data["model"] = f"MCP: {tool_name}" + model_metadata: Final[_MCPModelMetadata] = { + **(data.get("metadata") or MappingProxyType({})), + "model_group": f"MCP: {tool_name}", + } + data["metadata"] = model_metadata proxy_base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data) _request_start_time: Final = datetime.now() # noqa: DTZ005 # naive to match the tool start time below try: @@ -1226,6 +1253,8 @@ if MCP_AVAILABLE: if "metadata" in data and "user_api_key_auth" in data["metadata"]: data["user_api_key_auth"] = data["metadata"]["user_api_key_auth"] + _stamp_mcp_tool_metadata(logging_obj, server_id, tool_name) + # Resolve allowed MCP servers with IP filtering ( allowed_mcp_servers, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index be06d2e7321..6c9e1511b00 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3320,6 +3320,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob # single-owner so its meaning stays trustworthy. mcp_session_resource_server_id: str | None = Field(default=None, exclude=True) mcp_toolset_id: str | None = Field(default=None, exclude=True) + authenticated_by_custom_auth: bool = Field(default=False, exclude=True) via_virtual_key: bool = Field( default=False, exclude=True, @@ -3381,6 +3382,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob values.pop("mcp_session_resource_server_id", None) values.pop("mcp_toolset_id", None) values.pop("via_virtual_key", None) + values.pop("authenticated_by_custom_auth", None) values.pop("agent_caller", None) values.pop("managed_agent_context", None) values.pop("managed_agent_policy", None) diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py index e94d5e7ea78..6ddcd20d919 100644 --- a/litellm/proxy/agent_endpoints/a2a_endpoints.py +++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py @@ -722,6 +722,8 @@ async def invoke_agent_a2a( detail=f"Agent '{agent_id}' is not allowed for your key/team. Contact proxy admin for access.", ) + user_api_key_dict.invoked_agent_id = agent.agent_id + _enforce_inbound_trace_id(agent, request) # Get backend URL and agent name @@ -759,6 +761,10 @@ async def invoke_agent_a2a( if "metadata" not in body: body["metadata"] = {} body["metadata"]["agent_id"] = agent.agent_id + body["metadata"]["model_group"] = f"a2a_agent/{agent_name}" + body["metadata"]["model_info"] = { # mutable-ok: request hooks mutate metadata before JSON logging + "id": agent.agent_id + } body["agent_id"] = agent.agent_id body.update( @@ -862,6 +868,7 @@ async def invoke_agent_a2a( # results written by the unified_guardrail hook are captured. logging_obj._defer_async_logging = True response = await asend_message( + model=f"a2a_agent/{agent_name}", request=a2a_request, api_base=agent_url, litellm_params=litellm_params, diff --git a/litellm/proxy/agent_endpoints/a2a_routing.py b/litellm/proxy/agent_endpoints/a2a_routing.py index 8a795214750..c57315ebc21 100644 --- a/litellm/proxy/agent_endpoints/a2a_routing.py +++ b/litellm/proxy/agent_endpoints/a2a_routing.py @@ -57,7 +57,7 @@ async def route_a2a_agent_request( user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value ) - if not is_admin: + if not is_admin or agent.identity_managed: is_allowed: Final = await AgentRequestHandler.is_agent_allowed( agent_id=agent.agent_id, user_api_key_auth=user_api_key_dict, diff --git a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py index 75b99b0ab79..4e8880d37c5 100644 --- a/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py +++ b/litellm/proxy/agent_endpoints/auth/agent_permission_handler.py @@ -177,11 +177,10 @@ class AgentRequestHandler: registered: Final = global_agent_registry.get_agent_by_id(agent_id) registry_managed: Final = isinstance(registered, AgentResponse) and registered.identity_managed - if registry_managed or (registered is None and prisma_client is not None): + if registry_managed or prisma_client is not None: target: Final = await AgentIdentityStore.from_client(prisma_client).agent(agent_id) if isinstance(target, AgentIdentityFailure): - if registry_managed: - raise_identity_failure(target) + raise_identity_failure(target) elif target is None and registry_managed: return False elif isinstance(target, AgentResponse) and target.identity_managed: @@ -200,6 +199,7 @@ class AgentRequestHandler: if key_hash and managed_agent_policy(user_api_key_auth) is None and not user_api_key_auth.is_session_token + and not user_api_key_auth.authenticated_by_custom_auth else user_api_key_auth ) fresh_auth: Final = authority.model_copy( @@ -678,14 +678,44 @@ async def _managed_actor_agent_access(auth: UserAPIKeyAuth) -> AgentAccess: return RestrictedAgentAccess(capped.intersection(human_ids)) -async def verified_human_agent_grants(user_id: str | None, team_id: str | None = None) -> frozenset[str]: +async def _verified_human_agent_sources( + user_id: str | None, *, allowed_team_ids: frozenset[str] | None = None +) -> tuple[tuple[str | None, frozenset[str]], ...]: from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler if user_id is None: - return frozenset() + return () human: Final = await MCPRequestHandler.reload_admitted_user(user_id, requires_fresh_policy=True) - sources: Final = await MCPRequestHandler.admitted_subject_sources( - human, allowed_team_ids=frozenset((team_id,)) if team_id else frozenset() + sources: Final = await MCPRequestHandler.admitted_subject_sources(human, allowed_team_ids=allowed_team_ids) + access: Final = await asyncio.gather(*(_strict_agent_access(source) for source in sources)) + return tuple((source.team_id, _granted_ids(grant)) for source, grant in zip(sources, access, strict=True)) + + +async def verified_human_agent_grants(user_id: str | None, team_id: str | None = None) -> frozenset[str]: + sources: Final = await _verified_human_agent_sources( + user_id, allowed_team_ids=frozenset((team_id,)) if team_id else frozenset() ) - human_access: Final = await asyncio.gather(*(_strict_agent_access(source) for source in sources)) - return frozenset().union(*(_granted_ids(access) for access in human_access)) + return frozenset().union(*(grants for source, grants in sources if source is None or source == team_id)) + + +async def resolve_delegated_agent_team( + user_id: str | None, + agent_id: str, + team_id: str | None, + *, + explicit_team: bool, + allowed_team_ids: frozenset[str] | None = None, +) -> str | None: + sources: Final = await _verified_human_agent_sources(user_id) + if any(source is None and agent_id in grants for source, grants in sources): + return team_id + granting_teams: Final = frozenset( + source + for source, grants in sources + if source is not None and agent_id in grants and (allowed_team_ids is None or source in allowed_team_ids) + ) + if team_id in granting_teams: + return team_id + if not explicit_team and granting_teams: + return min(granting_teams) + raise HTTPException(403, "Select a team that grants access to this agent using x-litellm-team-id") diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index a15cf074ad5..17d988127ec 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -1,11 +1,129 @@ -from typing import Final +from collections.abc import Mapping +from itertools import product +from types import MappingProxyType +from typing import Annotated, Final, Literal -from litellm.proxy._types import UserAPIKeyAuth +from pydantic import Field, TypeAdapter, ValidationError + +from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth from litellm.proxy.agent_endpoints.identity_store import AgentIdentityStore from litellm.proxy.agent_endpoints.managed_identity import raise_identity_failure from litellm.types.agents import AgentResponse from litellm.types.proxy.agent_identity import AgentIdentityFailure, ManagedAgentContext +_MANAGED_REALTIME_ROUTES: Final = frozenset(("/realtime", "/v1/realtime", "/openai/v1/realtime")) +_MANAGED_MODEL_ROUTES: Final = frozenset( + f"{prefix}/{operation}" + for prefix, operation in product( + ("", "/v1"), + ( + "chat/completions", + "completions", + "embeddings", + "responses", + "messages", + "messages/count_tokens", + "images/generations", + "images/edits", + "audio/transcriptions", + "audio/speech", + "moderations", + "rerank", + "ocr", + ), + ) +) | frozenset( + ( + "/openai/v1/responses", + "/v2/rerank", + "/claude_code_gateway/v1/messages", + "/claude_code_gateway/v1/messages/count_tokens", + "/cursor/chat/completions", + ) +) +_MANAGED_MODEL_PATHS: Final = ( + "/engines/{model:path}/chat/completions", + "/engines/{model:path}/completions", + "/engines/{model:path}/embeddings", + "/openai/deployments/{model:path}/chat/completions", + "/openai/deployments/{model:path}/completions", + "/openai/deployments/{model:path}/embeddings", + "/openai/deployments/{model:path}/images/generations", + "/openai/deployments/{model:path}/images/edits", + "/v1beta/models/{model_name:path}:countTokens", + "/v1beta/models/{model_name:path}:generateContent", + "/v1beta/models/{model_name:path}:streamGenerateContent", + "/models/{model_name:path}:countTokens", + "/models/{model_name:path}:generateContent", + "/models/{model_name:path}:streamGenerateContent", +) +_MANAGED_MCP_ROUTES: Final = tuple( + route for route in LiteLLMRoutes.mcp_inference_routes.value if route not in ("/token", "/introspect") +) + + +_MODEL_ROUTE_KINDS: Final[ + Mapping[str, Literal["image_generation", "image_edit", "moderation", "speech", "body", "path"]] +] = MappingProxyType( + { + "/images/generations": "image_generation", + "/images/edits": "image_edit", + "/moderations": "moderation", + "/audio/transcriptions": "moderation", + "/audio/speech": "speech", + "/rerank": "body", + "/messages/count_tokens": "body", + ":countTokens": "path", + } +) + + +def managed_agent_route_allowed(route: str, method: str | None) -> bool: + from litellm.proxy.auth.route_checks import RouteChecks + + if route in ("/agents", "/v1/agents"): + return method in (None, "GET", "HEAD") + if route in _MANAGED_REALTIME_ROUTES: + return method in (None, "GET") + if route in _MANAGED_MODEL_ROUTES or RouteChecks.check_route_access(route, _MANAGED_MODEL_PATHS): + return method in (None, "POST") + return RouteChecks.check_route_access(route, _MANAGED_MCP_ROUTES) or RouteChecks.check_route_access( + route, LiteLLMRoutes.agent_inference_routes.value + ) + + +def managed_inference_request( + route: str, + body: Mapping[str, object], + settings: Mapping[str, object], + cli_model: str | None, + path_model: object = None, + query_model: object = None, +) -> dict[str, object]: + from litellm.proxy.auth.route_checks import RouteChecks + + if route in _MANAGED_REALTIME_ROUTES: + model: Final = query_model or body.get("model") + if not isinstance(model, str) or not model: + raise_identity_failure( + AgentIdentityFailure(message="Managed inference requires an explicit or configured model") + ) + return {**body, "model": model} # mutable-ok: centralized auth hooks add request tags and budget metadata + if route not in _MANAGED_MODEL_ROUTES and not RouteChecks.check_route_access(route, _MANAGED_MODEL_PATHS): + return dict(body) # mutable-ok: centralized auth hooks add request tags and budget metadata + from litellm.proxy.common_utils.http_parsing_utils import resolve_inference_model + + kind: Final = next((kind for suffix, kind in _MODEL_ROUTE_KINDS.items() if route.endswith(suffix)), "completion") + endpoint_model: Final = path_model or ( + query_model if route.endswith(("/completions", "/embeddings", "/images/generations", "/images/edits")) else None + ) + effective: Final = resolve_inference_model(body.get("model"), settings, cli_model, endpoint_model, kind=kind) + if not isinstance(effective, str) or not effective: + raise_identity_failure( + AgentIdentityFailure(message="Managed inference requires an explicit or configured model") + ) + return {**body, "model": effective} # mutable-ok: centralized auth hooks add request tags and budget metadata + def managed_agent_policy(auth: "UserAPIKeyAuth | None") -> AgentResponse | None: """The admitted managed policy, or ``None`` when the subject was never admitted as a managed agent. @@ -82,3 +200,53 @@ def actor_admission_failure( if context.mode == "delegated" and not context.user_id: return AgentIdentityFailure(message="A verified human subject is required") return None + + +_INVOCATION_COST: Final = TypeAdapter(Annotated[float, Field(ge=0, allow_inf_nan=False)]) + + +def invocation_target(route: str, body: Mapping[str, object]) -> str | None: + components: Final = tuple(route.strip("/").split("/")) + path: Final = components[1:] if components and components[0] == "v1" else components + if len(path) >= 2 and path[0] == "a2a": + return path[1] or None + model: Final = body.get("model") + return model.removeprefix("a2a/") or None if isinstance(model, str) and model.startswith("a2a/") else None + + +async def prepare_agent_invocation( + auth: UserAPIKeyAuth, target_name: str, store: AgentIdentityStore | None, *, billable: bool = True +) -> None: + from litellm.proxy.agent_endpoints.auth.agent_permission_handler import AgentRequestHandler + from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through + + registered: Final = await get_agent_with_read_through(target_name) + if registered is None: + return + registered_managed: Final = registered.identity_managed or registered.identity is not None + if store is None and registered_managed: + raise_identity_failure( + AgentIdentityFailure(code="policy_unavailable", message="Managed agent policy requires a database") + ) + target: Final = await store.agent(registered.agent_id) if store is not None else None + if isinstance(target, AgentIdentityFailure): + raise_identity_failure(target) + if target is None and registered_managed: + raise_identity_failure(AgentIdentityFailure(message="Invoked agent no longer exists")) + effective: Final = target if target is not None else registered + if not effective.identity_managed and auth.managed_agent_policy is None: + return + if not await AgentRequestHandler.is_agent_allowed(effective.agent_id, auth): + raise_identity_failure(AgentIdentityFailure(message="The caller is not permitted to invoke this agent")) + auth.invoked_agent_id = effective.agent_id + auth.invoked_agent_policy = effective + if auth.agent_id is None and effective.identity_managed: + auth.billing_agent_policy = effective + raw_fee: Final = (effective.litellm_params or MappingProxyType({})).get("cost_per_query", 0.0) if billable else 0.0 + try: + fee: Final = _INVOCATION_COST.validate_python(raw_fee) + except ValidationError: + raise_identity_failure( + AgentIdentityFailure(code="policy_unavailable", message="Agent invocation price is invalid") + ) + auth.agent_invocation_cost = fee diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 4f41b283a33..4448d860217 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -52,6 +52,10 @@ from litellm.proxy._types import ( TeamMemberAddRequest, UserAPIKeyAuth, ) +from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_route_allowed +from litellm.proxy.agent_endpoints.identity import has_legacy_identity +from litellm.proxy.agent_endpoints.identity_store import AgentIdentityStore, resolve_managed_agent +from litellm.proxy.agent_endpoints.managed_identity import raise_identity_failure from litellm.proxy.auth.auth_checks import can_team_access_model from litellm.proxy.auth.model_access_denied import ( ModelAccessDeniedHTTPException, @@ -67,6 +71,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.user_repository import UserRepository from litellm.types.agents import AgentResponse +from litellm.types.proxy.agent_identity import AgentIdentityFailure from litellm.types.proxy.auth.auth_checks import UserNotFoundError from .auth_checks import ( @@ -157,6 +162,8 @@ class HeaderTeam: class AgentLookup(Protocol): """The registered-agent lookups a JWT agent claim is matched against.""" + def get_agent_list(self) -> Sequence[AgentResponse]: ... + def get_agent_by_id(self, agent_id: str) -> AgentResponse | None: """The agent registered under ``agent_id``, if any.""" @@ -167,6 +174,9 @@ class AgentLookup(Protocol): class _NoRegisteredAgents: """The lookup in force until the proxy binds its agent registry: no agent is registered, so no claim matches.""" + def get_agent_list(self) -> tuple[AgentResponse, ...]: + return () + def get_agent_by_id(self, agent_id: str) -> None: return None @@ -398,7 +408,7 @@ class JWTHandler: return [] - def get_all_jwt_team_ids(self, token: dict) -> list[str]: + def get_all_jwt_team_ids(self, token: dict[str, object]) -> list[str]: """ Return team IDs from both the plural ``team_ids_jwt_field`` and the singular ``team_id_jwt_field`` claim (string or list of strings), as a @@ -522,7 +532,7 @@ class JWTHandler: team_id = default_value return team_id - def get_team_alias(self, token: dict, default_value: str | None) -> str | None: + def get_team_alias(self, token: dict[str, object], default_value: str | None) -> str | None: """ Extract team name/alias from JWT token using the configured team_alias_jwt_field. @@ -1096,6 +1106,15 @@ class JWTHandler: "options": options or None, } + def managed_issuer_is_trusted(self, issuer: object) -> bool: + if not isinstance(issuer, str): + return False + configured: Final = self.litellm_jwtauth.issuers or () + for item in configured: + if item.issuer == issuer: + return bool(item.audience) and not item.disable_audience_validation + return issuer == os.getenv("JWT_ISSUER") and bool(os.getenv("JWT_AUDIENCE")) + def _get_configured_issuer(self, token: str) -> JWTIssuerConfig | None: litellm_jwtauth: Final[_JWTAuthSettings | None] = getattr(self, "litellm_jwtauth", None) if litellm_jwtauth is None: @@ -1488,7 +1507,12 @@ class JWTAuthManager: agent: Final = agent_registry.get_agent_by_id(agent_id=agent_claim) or agent_registry.get_agent_by_name( agent_name=agent_claim ) - if agent is None: + if ( + agent is None + or agent.identity_managed + or agent.identity is not None + or has_legacy_identity(agent.litellm_params) + ): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"No registered agent matches JWT claim {jwt_handler.litellm_jwtauth.agent_id_jwt_field}={agent_claim}", @@ -2159,7 +2183,7 @@ class JWTAuthManager: parent_otel_span: Span | None, proxy_logging_obj: ProxyLogging, team_id_upsert: bool | None, - ) -> tuple: + ) -> tuple[str | None, LiteLLM_TeamTable | None, LiteLLM_TeamMembership | None]: """ If JWT did not resolve team_id, but the user belongs to exactly one team in LiteLLM, load that team (and membership when user_id is set) so that @@ -2478,12 +2502,39 @@ class JWTAuthManager: """Resolve and authorize JWT context; only normal admission supplies provisioning.""" handler: Final = jwt_handler jwt_valid_token: Final = await JWTAuthManager.authenticate_jwt(api_key, handler) + managed: Final = await resolve_managed_agent(jwt_valid_token, prisma_client, cache=user_api_key_cache) + if managed is not None: + if not handler.managed_issuer_is_trusted(jwt_valid_token.get("iss")): + raise HTTPException(403, "Managed agents require trusted JWT issuer and audience validation") + if not managed_agent_route_allowed(route, request_method): + raise HTTPException(403, "Agent identities can only access inference and agent discovery routes") + evidence: Final = await AgentIdentityStore.from_client(prisma_client).record_authentication(managed) + if isinstance(evidence, AgentIdentityFailure): + raise_identity_failure(evidence) + if managed.mode == "autonomous": + return JWTAuthBuilderResult( + is_proxy_admin=False, + team_id=None, + team_object=None, + user_id=None, + user_email=None, + user_object=None, + org_id=None, + org_object=None, + end_user_id=None, + end_user_object=None, + token=api_key, + team_membership=None, + jwt_claims=jwt_valid_token, + agent_id=managed.agent_id, + managed_agent_context=managed, + ) team_id_upsert: Final = provisioning.team_id_upsert if provisioning is not None else False model: Final = request_data.get("model") requested_model: Final = model if isinstance(model, str) else None # Check RBAC - rbac_role: Final = handler.get_rbac_role(token=jwt_valid_token) + rbac_role: Final = handler.get_rbac_role(token=jwt_valid_token) if managed is None else None await JWTAuthManager.check_rbac_role(handler, jwt_valid_token, general_settings, request_data, route, rbac_role) # Check Scope Based Access @@ -2499,7 +2550,11 @@ class JWTAuthManager: object_id = handler.get_object_id(token=jwt_valid_token, default_value=None) # Get basic user info - user_id, user_email, valid_user_email = await JWTAuthManager.get_user_info(handler, jwt_valid_token) + user_id, user_email, valid_user_email = ( + (managed.user_id, None, None) + if managed is not None + else await JWTAuthManager.get_user_info(handler, jwt_valid_token) + ) # Get IDs org_id: Final = handler.get_org_id(token=jwt_valid_token, default_value=None) @@ -2514,23 +2569,31 @@ class JWTAuthManager: elif rbac_role == LitellmUserRoles.INTERNAL_USER: user_id = object_id - agent_id: Final = JWTAuthManager.resolve_agent_id( - jwt_handler=handler, - jwt_valid_token=jwt_valid_token, - agent_registry=handler.agent_lookup, + agent_id: Final = ( + managed.agent_id + if managed is not None + else JWTAuthManager.resolve_agent_id( + jwt_handler=handler, + jwt_valid_token=jwt_valid_token, + agent_registry=handler.agent_lookup, + ) ) # Check admin access - admin_result: Final = await JWTAuthManager.check_admin_access( - handler, - scopes, - route, - user_id, - org_id, - api_key, - jwt_valid_token, - user_email=user_email, - agent_id=agent_id, + admin_result: Final = ( + None + if managed is not None + else await JWTAuthManager.check_admin_access( + handler, + scopes, + route, + user_id, + org_id, + api_key, + jwt_valid_token, + user_email=user_email, + agent_id=agent_id, + ) ) if admin_result: await JWTAuthManager._attach_team_from_header_for_admin( @@ -2673,8 +2736,47 @@ class JWTAuthManager: team_id_upsert=team_id_upsert, ) - if team_id and not JWTAuthManager._team_has_passthrough_route_access( - team_object=team_object, + from litellm.proxy.agent_endpoints.auth.agent_permission_handler import resolve_delegated_agent_team + + claimed_teams: Final[frozenset[str]] = ( + frozenset(handler.get_all_jwt_team_ids(jwt_valid_token)) if managed is not None else frozenset() + ) + scoped_teams: Final[frozenset[str] | None] = claimed_teams or ( + frozenset((team_id,)) + if managed is not None and team_id and handler.get_team_alias(jwt_valid_token, default_value=None) + else None + ) + granting_team: Final = ( + await resolve_delegated_agent_team( + managed.user_id, + managed.agent_id, + team_id, + explicit_team=header_team is not None, + allowed_team_ids=None if handler.litellm_jwtauth.fallback_to_db_teams else scoped_teams, + ) + if managed is not None + else team_id + ) + if granting_team is not None and granting_team != team_id: + if not JWTAuthManager._is_team_route_allowed(route, request_method, handler): + raise HTTPException(403, "The granting team is not allowed to access this route") + + selected_team_id: Final[str | None] = granting_team if granting_team is not None else team_id + selected_team_object: Final[LiteLLM_TeamTable | None] = ( + await get_team_object( + team_id=selected_team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + check_db_only=True, + ) + if selected_team_id is not None and selected_team_id != team_id + else team_object + ) + + if selected_team_id and not JWTAuthManager._team_has_passthrough_route_access( + team_object=selected_team_object, route=route, request_method=request_method, team_allowed_routes=handler.litellm_jwtauth.team_allowed_routes, @@ -2696,7 +2798,7 @@ class JWTAuthManager: user_email=user_email, org_id=org_id, end_user_id=end_user_id, - team_id=team_id, + team_id=selected_team_id, valid_user_email=valid_user_email, jwt_handler=handler, prisma_client=prisma_client, @@ -2705,13 +2807,13 @@ class JWTAuthManager: proxy_logging_obj=proxy_logging_obj, route=route, org_alias=org_alias, - user_id_upsert=provisioning.user_id_upsert if provisioning is not None else False, + user_id_upsert=provisioning.user_id_upsert if provisioning is not None and managed is None else False, ) # Derive org_id from org_object if resolved by alias resolved_org_id: Final = org_object.organization_id if org_object else org_id - if provisioning is not None: + if provisioning is not None and managed is None: await JWTAuthManager.sync_user_role_and_teams( jwt_handler=handler, jwt_valid_token=jwt_valid_token, @@ -2721,7 +2823,7 @@ class JWTAuthManager: ) # If JWT did not resolve team_id, attempt a team fallback. - if team_id is None and db_team_fallback: + if selected_team_id is None and db_team_fallback: ( team_id, team_object, @@ -2750,7 +2852,7 @@ class JWTAuthManager: team_allowed_routes=handler.litellm_jwtauth.team_allowed_routes, ): JWTAuthManager._raise_team_passthrough_route_denial(route=route) - elif team_id is None: + elif selected_team_id is None: ( team_id, team_object, @@ -2764,9 +2866,9 @@ class JWTAuthManager: proxy_logging_obj=proxy_logging_obj, team_id_upsert=team_id_upsert, ) - elif provisional_header_team is not None and team_id == provisional_header_team.team_id: + elif provisional_header_team is not None and selected_team_id == provisional_header_team.team_id: JWTAuthManager._validate_header_team_in_db_membership( - team_id=team_id, + team_id=selected_team_id, user_object=user_object, header_value=provisional_header_team.header_value, ) @@ -2783,28 +2885,35 @@ class JWTAuthManager: ), ) + authorized_team_id: Final[str | None] = selected_team_id if selected_team_id is not None else team_id + authorized_team_object: Final[LiteLLM_TeamTable | None] = ( + selected_team_object if selected_team_id is not None else team_object + ) + ## MAP USER TO TEAMS - if provisioning is not None: + if provisioning is not None and managed is None: await JWTAuthManager.map_user_to_teams( user_object=user_object, - team_object=team_object, + team_object=authorized_team_object, ) # Validate that a valid rbac id is returned for spend tracking JWTAuthManager.validate_object_id( user_id=user_id, - team_id=team_id, + team_id=authorized_team_id, enforce_rbac=bool(general_settings.get("enforce_rbac", False)), is_proxy_admin=False, ) # check if user is proxy admin - is_proxy_admin: Final = bool(user_object and user_object.user_role == LitellmUserRoles.PROXY_ADMIN) + is_proxy_admin: Final = managed is None and bool( + user_object and user_object.user_role == LitellmUserRoles.PROXY_ADMIN + ) return JWTAuthBuilderResult( is_proxy_admin=is_proxy_admin, - team_id=team_id, - team_object=team_object, + team_id=authorized_team_id, + team_object=authorized_team_object, user_id=user_id, user_email=(user_object.user_email if user_object is not None and user_object.user_email else user_email), user_object=user_object, @@ -2816,6 +2925,7 @@ class JWTAuthManager: team_membership=team_membership_object, jwt_claims=jwt_valid_token, agent_id=agent_id, + managed_agent_context=managed, ) @staticmethod @@ -2826,11 +2936,13 @@ class JWTAuthManager: """Keep JWT identity and permission attribution identical across consumers.""" user: Final = result["user_object"] admin: Final = result["is_proxy_admin"] - return UserAPIKeyAuth( + auth: Final = UserAPIKeyAuth( api_key=None, user_role=( LitellmUserRoles.PROXY_ADMIN if admin + else LitellmUserRoles.INTERNAL_USER + if result.get("managed_agent_context") is not None else LitellmUserRoles(user.user_role) if user is not None and user.user_role is not None else LitellmUserRoles.INTERNAL_USER @@ -2852,3 +2964,8 @@ class JWTAuthManager: user_id=result["user_id"], ), ) + auth.managed_agent_context = result.get("managed_agent_context") + auth._managed_delegation_verified = ( # pyright: ignore[reportPrivateUsage] # JWT admission produces the one-shot proof consumed by managed authorization + auth.managed_agent_context is not None and auth.managed_agent_context.mode == "delegated" + ) + return auth diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 36c6a4c476b..5194f62cf78 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -655,6 +655,8 @@ async def user_api_key_auth_websocket_for_model(websocket: WebSocket, model: str # never reaches the fallback. synthetic_scope: Final[dict[str, Any]] = { "type": "http", + "method": "GET", + "query_string": ws_scope.get("query_string", b""), "headers": scope_headers, "path": ws_scope.get("path", ""), "state": ws_scope.setdefault("state", {}), # mutable-ok: Starlette's socket state, shared with the request @@ -1559,6 +1561,7 @@ async def _user_api_key_auth_builder( route=route, parent_otel_span=parent_otel_span, ) + validated.authenticated_by_custom_auth = True return validated elif response is not None and isinstance(response, str): api_key = response @@ -1574,6 +1577,7 @@ async def _user_api_key_auth_builder( route=route, parent_otel_span=parent_otel_span, ) + validated.authenticated_by_custom_auth = True return validated ### LITELLM-DEFINED AUTH FUNCTION ### @@ -1656,6 +1660,16 @@ async def _user_api_key_auth_builder( else: jwt_claims = await jwt_handler.auth_jwt(token=api_key) + from litellm.proxy.agent_endpoints.identity_store import resolve_managed_agent + + if ( + jwt_claims + and await resolve_managed_agent(jwt_claims, prisma_client, cache=user_api_key_cache) is not None + ): + raise HTTPException( + 403, "Managed agents require direct JWT authentication without virtual-key mapping" + ) + resolve_result: Final = await _resolve_jwt_to_virtual_key( jwt_claims=jwt_claims, jwt_handler=jwt_handler, @@ -3130,7 +3144,10 @@ async def _reserve_budget_after_common_checks( end_user_id=end_user_id, end_user_object=end_user_object, apply_user_budget_to_team_keys=general_settings.get("apply_user_budget_to_team_keys") is True, - fail_closed_budget_enforcement=general_settings.get("fail_closed_budget_enforcement") is True, + fail_closed_budget_enforcement=( + general_settings.get("fail_closed_budget_enforcement") is True + or user_api_key_auth_obj.billing_agent_policy is not None + ), raw_body=await read_raw_json_body(request=request), ) if request is not None: @@ -3204,19 +3221,48 @@ async def _authorize_authenticated_request( # admin-only-route / model-access / budget checks) surface as # ProxyException consistently with pre-refactor behavior. try: - from litellm.proxy.agent_endpoints.auth.managed_authorization import admit_managed_actor + from litellm.proxy.agent_endpoints.auth.managed_authorization import ( + admit_managed_actor, + invocation_target, + managed_agent_route_allowed, + managed_inference_request, + prepare_agent_invocation, + ) from litellm.proxy.agent_endpoints.identity_store import AgentIdentityStore - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import general_settings, prisma_client, user_model + store: Final = AgentIdentityStore.from_client(prisma_client) if prisma_client is not None else None if user_api_key_auth_obj.agent_id is not None: - await admit_managed_actor( + await admit_managed_actor(user_api_key_auth_obj, store) + if user_api_key_auth_obj.managed_agent_policy is not None and not managed_agent_route_allowed( + route, request.method + ): + raise HTTPException(403, "Agent identities can only access inference and agent discovery routes") + authorized_data: Final = ( + managed_inference_request( + route, + request_data, + general_settings, + user_model, + request.path_params.get("model") or request.path_params.get("model_name"), + request.query_params.get("model"), + ) + if user_api_key_auth_obj.managed_agent_policy is not None + else request_data + ) + target_name: Final = invocation_target(route, authorized_data) + if target_name is not None: + await prepare_agent_invocation( user_api_key_auth_obj, - AgentIdentityStore.from_client(prisma_client) if prisma_client is not None else None, + target_name, + store, + billable=request_data.get("method") + in (None, "message/send", "message/stream", "SendMessage", "SendStreamingMessage"), ) await _run_centralized_common_checks( user_api_key_auth_obj=user_api_key_auth_obj, request=request, - request_data=request_data, + request_data=authorized_data, route=route, ) except Exception as e: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 10724e9e7e6..e7a08711eb1 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -93,6 +93,7 @@ from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_bod from litellm.proxy.common_utils.http_parsing_utils import ( get_client_requested_model, get_tags_from_request_body, + resolve_inference_model, ) from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, @@ -2068,11 +2069,12 @@ class ProxyBaseLLMRequestProcessing: if isinstance(model, str): reject_url_valued_destination("model", model) - self.data["model"] = ( - general_settings.get("completion_model", None) # server default - or user_model # model name passed via cli args - or model # for azure deployments - or self.data.get("model", None) # default passed in http request + self.data["model"] = resolve_inference_model( + self.data.get("model"), + general_settings, + user_model, + model, + kind="image_edit" if route_type == "aimage_edit" else "completion", ) # override with user settings, these are params passed via cli diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 1c2bd7ea217..f436bc96dbd 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -2,11 +2,11 @@ import json import re from collections.abc import Collection, Mapping from types import MappingProxyType, UnionType -from typing import Annotated, Any, Final, Union, get_args, get_origin +from typing import Annotated, Any, Final, Literal, Union, get_args, get_origin import orjson from fastapi import Request, UploadFile, status -from typing_extensions import NotRequired, ReadOnly, Required +from typing_extensions import NotRequired, ReadOnly, Required, assert_never from litellm._logging import verbose_proxy_logger from litellm.constants import ( @@ -25,6 +25,40 @@ _FORM_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-www-form- _ANNOTATION_QUALIFIERS: Final[frozenset[object]] = frozenset({Annotated, NotRequired, ReadOnly, Required}) +def resolve_inference_model( + body_model: object, + settings: Mapping[str, object], + cli_model: str | None, + endpoint_model: object = None, + *, + kind: Literal[ + "completion", "image_generation", "image_edit", "moderation", "speech", "body", "path" + ] = "completion", +) -> object: + match kind: + case "image_generation": + return cli_model or endpoint_model or settings.get("image_generation_model") or body_model + case "image_edit": + return ( + settings.get("completion_model") + or cli_model + or endpoint_model + or settings.get("image_generation_model") + or body_model + ) + case "moderation": + return cli_model or settings.get("moderation_model") or body_model + case "speech": + return cli_model or body_model + case "body": + return body_model + case "path": + return endpoint_model + case "completion": + return settings.get("completion_model") or cli_model or endpoint_model or body_model + return assert_never(kind) + + def _normalize_media_type(content_type: str) -> str: """Return the bare media type per RFC 7231: strip params, trim, lowercase.""" if not content_type: diff --git a/litellm/proxy/common_utils/registry_read_through.py b/litellm/proxy/common_utils/registry_read_through.py index 63da5d15207..8a82e253c5c 100644 --- a/litellm/proxy/common_utils/registry_read_through.py +++ b/litellm/proxy/common_utils/registry_read_through.py @@ -174,7 +174,10 @@ async def _resync_agents(agent_id_or_name: str) -> bool: table: Final = agents_table(prisma_client) id_filter: Final[LiteLLM_AgentsTableWhereUniqueInput] = {"agent_id": agent_id_or_name} name_filter: Final[LiteLLM_AgentsTableWhereUniqueInput] = {"agent_name": agent_id_or_name} - include_permission: Final[LiteLLM_AgentsTableInclude] = {"object_permission": True} + include_permission: Final[LiteLLM_AgentsTableInclude] = { + "object_permission": True, + "identity": True, + } async with AGENT_RECONCILE_LOCK: if _agent_from_registry(agent_id_or_name) is not None: return True diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index e509f03d458..36896bd6d44 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -3065,6 +3065,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self, agent_id: str, data: dict, + policy: "AgentResponse | None" = None, ) -> list[RateLimitDescriptor]: """ Create rate limit descriptors for agent-level and session-level limits. @@ -3074,7 +3075,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): """ descriptors: Final[list[RateLimitDescriptor]] = [] - agent: Final = self._get_agent_from_registry(agent_id) + agent: Final = policy if policy is not None else self._get_agent_from_registry(agent_id) if agent is None: return descriptors @@ -3269,14 +3270,19 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptors=descriptors, ) - # Agent-level and session-level rate limits resolved_agent_id: Final = self._get_resolved_agent_id(user_api_key_dict, data) - - if resolved_agent_id: + for agent_id in dict.fromkeys((resolved_agent_id, user_api_key_dict.invoked_agent_id)): + if agent_id is None: + continue descriptors.extend( self._create_agent_rate_limit_descriptors( - agent_id=resolved_agent_id, + agent_id=agent_id, data=data, + policy=( + user_api_key_dict.managed_agent_policy + if agent_id == user_api_key_dict.agent_id + else user_api_key_dict.invoked_agent_policy + ), ) ) @@ -4965,6 +4971,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): tpm_limited_tags=stash.tpm_limited_tags if stash is not None else frozenset(), model_group=reconcile_model.group if reconcile_model is not None else None, ) + targets.extend( + scope + for scope in sorted(reserved_scopes) + if scope[0] in ("agent", "agent_session") and scope not in targets + ) charged_targets: Final = ( [target for target in targets if target[0] != "model_per_team"] if self._key_owns_model_tpm_limit_from_request_metadata(request_metadata, reconcile_model) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 877dbfabe5d..6a2ec120060 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -360,6 +360,7 @@ class _ProxyDBLogger(CustomLogger): team_id=team_id, end_user_id=end_user_id, call_type=call_type, + agent_id=metadata.get("billing_agent_id") or metadata.get("agent_id"), ): ## UPDATE DATABASE charged: Final = await _update_database_and_spend_counters( @@ -621,6 +622,7 @@ def _should_track_cost_callback( team_id: str | None, end_user_id: str | None, call_type: str | None = None, + agent_id: str | None = None, ) -> bool: """ Determine if the cost callback should be tracked based on the kwargs @@ -637,7 +639,13 @@ def _should_track_cost_callback( if ProxyUpdateSpend.disable_spend_updates() is True: return False - if user_api_key is not None or user_id is not None or team_id is not None or end_user_id is not None: + if ( + agent_id is not None + or user_api_key is not None + or user_id is not None + or team_id is not None + or end_user_id is not None + ): return True return call_type in _UNATTRIBUTED_TRACKABLE_CALL_TYPES diff --git a/litellm/proxy/image_endpoints/endpoints.py b/litellm/proxy/image_endpoints/endpoints.py index b9580ba3948..16dc38575da 100644 --- a/litellm/proxy/image_endpoints/endpoints.py +++ b/litellm/proxy/image_endpoints/endpoints.py @@ -21,6 +21,7 @@ from litellm.proxy.common_request_processing import ( from litellm.proxy.common_utils.http_parsing_utils import ( coerce_numeric_form_fields, numeric_form_fields, + resolve_inference_model, ) from litellm.proxy.common_utils.openai_error_payload import ( error_status_code, @@ -118,14 +119,9 @@ async def image_generation( if isinstance(model, str): reject_url_valued_destination("model", model) - data["model"] = ( - model - or general_settings.get("image_generation_model", None) # server default - or user_model # model name passed via cli args - or data.get("model", None) # default passed in http request + data["model"] = resolve_inference_model( + data.get("model"), general_settings, user_model, model, kind="image_generation" ) - if user_model: - data["model"] = user_model ### MODEL ALIAS MAPPING ### # check if model name in model alias map @@ -324,12 +320,6 @@ async def image_edit_api( if "prompt" not in data: data["prompt"] = None - data["model"] = ( - model - or general_settings.get("image_generation_model", None) # server default - or user_model # model name passed via cli args - or data.get("model", None) # default passed in http request - ) ######################################################### # Process request ######################################################### @@ -346,7 +336,7 @@ async def image_edit_api( general_settings=general_settings, proxy_config=proxy_config, select_data_generator=select_data_generator, - model=None, + model=model, user_model=user_model, user_temperature=user_temperature, user_request_timeout=user_request_timeout, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d48451de6b1..4188e8ad58a 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1664,7 +1664,19 @@ class LiteLLMProxyRequestSetup: _key_agent_id: Final = getattr(user_api_key_dict, "agent_id", None) _existing_agent_id: Final = data[_metadata_variable_name].get("agent_id") _resolved_agent_id: Final = _key_agent_id or _existing_agent_id - data[_metadata_variable_name]["agent_id"] = _resolved_agent_id + data[_metadata_variable_name]["agent_id"] = user_api_key_dict.invoked_agent_id or _resolved_agent_id + managed_context: Final = user_api_key_dict.managed_agent_context + data[_metadata_variable_name].update( + MappingProxyType( + { + "actor_agent_id": user_api_key_dict.agent_id, + "target_agent_id": user_api_key_dict.invoked_agent_id, + "billing_agent_id": user_api_key_dict.agent_id or user_api_key_dict.invoked_agent_id, + "agent_execution_mode": managed_context.mode if managed_context else None, + "verified_human_user_id": managed_context.user_id if managed_context else None, + } + ) + ) data[_metadata_variable_name]["user_api_end_user_max_budget"] = getattr( user_api_key_dict, "end_user_max_budget", None diff --git a/litellm/proxy/management_endpoints/sso/agent_subject_enrollment.py b/litellm/proxy/management_endpoints/sso/agent_subject_enrollment.py new file mode 100644 index 00000000000..e548b9b7fa2 --- /dev/null +++ b/litellm/proxy/management_endpoints/sso/agent_subject_enrollment.py @@ -0,0 +1,49 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final +from uuid import UUID + +from litellm.types.proxy.agent_identity import MicrosoftInteractiveSubject + + +def microsoft_interactive_subject( + tenant: str | None, + response: Mapping[str, object], + endpoints: Mapping[str, str | None], +) -> MicrosoftInteractiveSubject | None: + if tenant is None: + return None + try: + tenant_id: Final = str(UUID(tenant)) + object_id: Final = response.get("id") + if not isinstance(object_id, str): + return None + oid: Final = str(UUID(object_id)) + except ValueError: + return None + expected: Final = MappingProxyType( + { + "MICROSOFT_AUTHORIZATION_ENDPOINT": f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/authorize", + "MICROSOFT_TOKEN_ENDPOINT": f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token", + "MICROSOFT_USERINFO_ENDPOINT": "https://graph.microsoft.com/v1.0/me", + } + ) + if any(value and value != expected.get(name) for name, value in endpoints.items()): + return None + return MicrosoftInteractiveSubject( + issuer=f"https://login.microsoftonline.com/{tenant_id}/v2.0", + tenant_id=tenant_id, + oid=oid, + ) + + +async def enroll_microsoft_subject(subject: object, user_id: object, client: object) -> None: + from litellm.proxy.agent_endpoints.identity_store import AgentIdentityStore + from litellm.proxy.agent_endpoints.managed_identity import raise_identity_failure + from litellm.types.proxy.agent_identity import AgentIdentityFailure + + if not isinstance(subject, MicrosoftInteractiveSubject) or not isinstance(user_id, str) or not user_id: + return + result: Final = await AgentIdentityStore.from_client(client).enroll_interactive_human(subject, user_id) + if isinstance(result, AgentIdentityFailure): + raise_identity_failure(result) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 618b200a14c..2a22077eb99 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -2313,6 +2313,11 @@ async def _complete_cli_sso_callback_session( status_code=500, detail="Could not resolve team model grants for this login. Please try again", ) + from litellm.proxy.management_endpoints.sso.agent_subject_enrollment import enroll_microsoft_subject + + await enroll_microsoft_subject( + request.scope.get("litellm_microsoft_interactive_subject"), user_info.user_id, prisma_client + ) resolved_teams: Final = _cli_sso_session_teams(team_details) attribution_metadata: Final = build_cli_sso_attribution_metadata(result=result) if attribution_metadata: @@ -3631,6 +3636,12 @@ class SSOAuthenticationHandler: }, ) + from litellm.proxy.management_endpoints.sso.agent_subject_enrollment import enroll_microsoft_subject + + await enroll_microsoft_subject( + request.scope.get("litellm_microsoft_interactive_subject"), user_id, prisma_client + ) + if isinstance(user_id, str) and user_id: await retain_sso_identity_assertion_for_ema(user_id=user_id, assertion=sso_assertion) await warn_if_id_jag_assertion_uncaptured(sso_assertion) @@ -4300,6 +4311,22 @@ class MicrosoftSSOHandler: original_msft_result["app_roles"] = app_roles return original_msft_result or {} + from litellm.proxy.management_endpoints.sso.agent_subject_enrollment import microsoft_interactive_subject + + request.scope["litellm_microsoft_interactive_subject"] = microsoft_interactive_subject( + microsoft_tenant, + original_msft_result, + MappingProxyType( + { + name: os.getenv(name) + for name in ( + "MICROSOFT_AUTHORIZATION_ENDPOINT", + "MICROSOFT_TOKEN_ENDPOINT", + "MICROSOFT_USERINFO_ENDPOINT", + ) + } + ), + ) result: Final = MicrosoftSSOHandler.openid_from_response( response=original_msft_result, team_ids=user_team_ids, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a994293a4d8..7200407b64c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -427,6 +427,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _safe_get_request_headers, check_file_size_under_limit, get_form_data, + resolve_inference_model, ) from litellm.proxy.common_utils.load_config_utils import get_config_from_bucket from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations @@ -12351,13 +12352,7 @@ async def moderations( proxy_config=proxy_config, ) - data["model"] = ( - general_settings.get("moderation_model", None) # server default - or user_model # model name passed via cli args - or data.get("model") # default passed in http request - ) - if user_model: - data["model"] = user_model + data["model"] = resolve_inference_model(data.get("model"), general_settings, user_model, kind="moderation") ### CALL HOOKS ### - modify incoming data / reject request before calling the model data = await proxy_logging_obj.pre_call_hook( @@ -12611,13 +12606,7 @@ async def audio_transcriptions( if data.get("user", None) is None and user_api_key_dict.user_id is not None: data["user"] = user_api_key_dict.user_id - data["model"] = ( - general_settings.get("moderation_model", None) # server default - or user_model # model name passed via cli args - or data.get("model", None) # default passed in http request - ) - if user_model: - data["model"] = user_model + data["model"] = resolve_inference_model(data.get("model"), general_settings, user_model, kind="moderation") router_model_names: Final = llm_router.model_names if llm_router is not None else [] diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 728579db5fc..01a216b61b5 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -77,6 +77,11 @@ _SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" _SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key" _MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" _AGENT_CALL_TYPE_SQL: Final = "'asend_message'" +_SESSION_REPRESENTATIVE_ORDER_SQL: Final = ( + f"(call_type = {_AGENT_CALL_TYPE_SQL}) DESC, " + f'CASE WHEN call_type = {_AGENT_CALL_TYPE_SQL} THEN "endTime" END DESC NULLS LAST, ' + f'call_type IN {_MCP_CALL_TYPES_SQL}, "startTime" DESC, request_id' +) _BATCH_CALL_TYPES_SQL: Final = "('acreate_batch', 'create_batch', 'aretrieve_batch', 'retrieve_batch')" _SPAN_TYPE_SQL_CONDITIONS: Final[Mapping[str, str]] = MappingProxyType( { @@ -2879,7 +2884,7 @@ async def ui_view_spend_logs( p += 1 # Status filter - if status_filter is not None: + if status_filter is not None and not (group_by_session is True and not is_search_lookup): if status_filter == "success": sql_conditions.append("(status = 'success' OR status IS NULL)") else: @@ -2925,6 +2930,23 @@ async def ui_view_spend_logs( sql_params.append(f"%{error_message}%") p += 1 + if status_filter is not None and group_by_session is True and not is_search_lookup: + session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE" + sql_conditions.append( + f"""({_SESSION_GROUP_KEY_SQL}) IN ( + SELECT session_key, api_key FROM ( + SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) + {_SESSION_KEY_EXPR} AS session_key, api_key, status + FROM "LiteLLM_SpendLogs" + WHERE {session_filter_conditions} + ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} + ) AS session_outcomes + WHERE COALESCE(status, 'success') = ${p} + )""" + ) + sql_params.append(status_filter) + p += 1 + if ( group_by_session is True and not is_v2 @@ -2991,7 +3013,7 @@ async def ui_view_spend_logs( {_SPEND_LOG_LIST_COLUMNS} FROM "LiteLLM_SpendLogs" WHERE {joined_conditions} - ORDER BY {_SESSION_GROUP_KEY_SQL}, call_type IN {_MCP_CALL_TYPES_SQL}, "startTime" DESC + ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} ) AS session_representatives ORDER BY {exact_request_id_first}{_order_expr} {_sql_dir}{_nulls_clause}, request_id LIMIT ${p} OFFSET ${p + 1} @@ -3063,7 +3085,7 @@ async def _fetch_session_representatives( next_param_index: int, session_keys: Sequence[tuple[str, str]], ) -> list[dict[str, object]]: # mutable-ok: _build_ui_spend_logs_response writes session counts onto each row - """Fetch the newest non-MCP row of each ``(session_key, api_key)`` session, in ``session_keys`` order.""" + """Fetch the final agent outcome, or newest non-MCP row, of each ``(session_key, api_key)`` session, in ``session_keys`` order.""" rep_query: Final = f""" SELECT * FROM ( SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) @@ -3073,7 +3095,7 @@ async def _fetch_session_representatives( AND ({_SESSION_GROUP_KEY_SQL}) IN ( SELECT * FROM unnest(${next_param_index}::text[], ${next_param_index + 1}::text[]) ) - ORDER BY {_SESSION_GROUP_KEY_SQL}, call_type IN {_MCP_CALL_TYPES_SQL}, "startTime" DESC + ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} ) AS session_representatives """ rep_rows: Final[Sequence[dict[str, object]]] = await _query_raw( # mutable-ok: rows are enriched in place @@ -3140,7 +3162,7 @@ async def _ui_session_grouped_spend_logs( page_size``, trimmed to the end of the ``SPEND_LOGS_PAGINATION_COUNT_CAP`` window the capped ``total`` promises, so a page never runs past that total and one starting at or past it returns no rows without a query. Each session is represented - by its newest non-MCP row, enriched by ``_build_ui_spend_logs_response`` + by its final agent outcome (or newest non-MCP row), enriched by ``_build_ui_spend_logs_response`` exactly like the flat listing, and the response carries ``next_session_cursor`` / ``has_more`` while ``total`` counts sessions (capped like the flat total). A page that runs out of sessions while still diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index e69c0c80420..81f583b419c 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -796,6 +796,7 @@ def get_logging_payload( model_id=_model_id, mcp_namespaced_tool_name=mcp_namespaced_tool_name, agent_id=agent_id, + billing_agent_id=clean_metadata.get("billing_agent_id"), requester_ip_address=clean_metadata.get("requester_ip_address", None), custom_llm_provider=custom_llm_provider or "", messages=_get_messages_for_spend_logs_payload( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_idp_token_exchange.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_idp_token_exchange.py index 03165bd0a4a..e94371a6056 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_idp_token_exchange.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_idp_token_exchange.py @@ -1,4 +1,5 @@ import logging +from typing import Final import pytest from fastapi import HTTPException @@ -235,3 +236,13 @@ async def test_a_database_fault_retrying_cannot_clear_is_not_reported_as_a_trans assert refusal == SubjectTokenRefusal(error="temporarily_unavailable", description=SUBJECT_TOKEN_CHECK_FAULTED) assert "retrying will not help" in refusal.description assert "faulted: " in caplog.text and "query engine binary not found" in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("user_id", [None, "delegating-user"]) +async def test_agent_token_cannot_be_exchanged_for_a_user_identity(user_id: str | None) -> None: + authorizer: Final = _Authorizer({**_authorized(user_id=user_id), "agent_id": "managed-agent"}) + result: Final = await _identity(authorizer) + assert isinstance(result, SubjectTokenRefusal) + assert result.error == "invalid_request" + assert "direct JWT authentication" in result.description diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index a7e56f3f84a..4cc7794d4ad 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -2244,7 +2244,7 @@ async def test_mcp_routing_chunked_initialize_to_stateful(): patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, - return_value=(MagicMock(), None, ["progress_test"], None, None, None), + return_value=(UserAPIKeyAuth(), None, ["progress_test"], None, None, None), ), patch( "litellm.proxy._experimental.mcp_server.server.set_auth_context", @@ -2356,7 +2356,7 @@ async def test_mcp_routing_caps_body_peek_for_oversized_chunked_body(): patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, - return_value=(MagicMock(), None, ["progress_test"], None, None, None), + return_value=(UserAPIKeyAuth(), None, ["progress_test"], None, None, None), ), patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), patch( @@ -2567,7 +2567,7 @@ async def test_mcp_routing_initialize_rejected_when_owner_at_session_cap(): patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, - return_value=(MagicMock(), None, ["progress_test"], None, None, None), + return_value=(UserAPIKeyAuth(), None, ["progress_test"], None, None, None), ), patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), patch( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py index ec6fdef69ee..eb1d8573ee3 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py @@ -9,9 +9,12 @@ they may send a stale `mcp-session-id` header. This test verifies that: import asyncio from unittest.mock import AsyncMock, MagicMock, patch -from litellm.types.mcp import MCPAuth + import pytest +from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.mcp import MCPAuth + class TestHandleStaleMcpSession: """Unit tests for the _handle_stale_mcp_session helper.""" @@ -260,7 +263,7 @@ async def test_stale_mcp_session_id_is_stripped(): patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, - return_value=(MagicMock(), None, None, None, None, None), + return_value=(UserAPIKeyAuth(), None, None, None, None, None), ), patch( "litellm.proxy._experimental.mcp_server.server.set_auth_context", @@ -337,7 +340,7 @@ async def test_delete_stale_mcp_session_returns_success(): patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, - return_value=(MagicMock(), None, None, None, None, None), + return_value=(UserAPIKeyAuth(), None, None, None, None, None), ), patch( "litellm.proxy._experimental.mcp_server.server.set_auth_context", @@ -386,7 +389,7 @@ async def test_failed_delete_preserves_stateful_session_tracking(): pytest.skip("MCP server not available") session_id = "delete-failure-session" - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.api_key = "sk-test" user_auth.user_id = "test-user" auth_context = MagicMock() @@ -491,7 +494,7 @@ async def test_valid_mcp_session_id_is_preserved(): patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, - return_value=(MagicMock(), None, None, None, None, None), + return_value=(UserAPIKeyAuth(), None, None, None, None, None), ), patch( "litellm.proxy._experimental.mcp_server.server.set_auth_context", @@ -554,7 +557,7 @@ async def test_no_mcp_session_id_header_works_normally(): patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, - return_value=(MagicMock(), None, None, None, None, None), + return_value=(UserAPIKeyAuth(), None, None, None, None, None), ), patch( "litellm.proxy._experimental.mcp_server.server.set_auth_context", @@ -613,7 +616,7 @@ async def test_per_user_oauth_missing_stored_token_returns_preemptive_401(): } receive = AsyncMock() send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = "test-user-id" oauth_server = MagicMock() oauth_server.auth_type = MCPAuth.oauth2 @@ -700,7 +703,7 @@ async def test_admitted_subject_missing_stored_token_challenged_with_resource_me } receive = AsyncMock() send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = "sso-user-42" user_auth.mcp_admitted_user_subject = True oauth_server = MagicMock() @@ -806,7 +809,7 @@ async def test_client_credentials_server_is_not_preemptively_challenged(m2m_fiel } ) send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = "test-user-id" m2m_server = MCPServer( server_id="m2m-server-id", @@ -892,7 +895,7 @@ async def test_handle_streamable_http_mcp_delegated_server_surfaces_upstream_cha } ) send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = None delegated_server = MagicMock() delegated_server.auth_type = MCPAuth.oauth2 @@ -996,7 +999,7 @@ async def test_per_user_oauth_with_stored_token_skips_preemptive_401(): } ) send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = "test-user-id" oauth_server = MagicMock() oauth_server.auth_type = MCPAuth.oauth2 @@ -1092,7 +1095,7 @@ async def test_handle_streamable_http_mcp_delegated_server_without_token_returns } ) send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = None delegated_server = MagicMock() delegated_server.auth_type = MCPAuth.oauth2 @@ -1192,7 +1195,7 @@ async def test_handle_streamable_http_mcp_token_exchange_without_subject_returns } ) send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = None obo_server = MagicMock() obo_server.auth_type = MCPAuth.oauth2_token_exchange @@ -1301,7 +1304,7 @@ async def test_handle_streamable_http_mcp_oauth_delegate_without_token_returns_g } ) send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = "u1" od_server = _build_passthrough_mode_server("od_server", MCPAuth.oauth_delegate) @@ -1366,7 +1369,7 @@ async def test_handle_streamable_http_mcp_oauth_delegate_with_forwarded_token_sk } ) send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = "u1" od_server = _build_passthrough_mode_server("od_server", MCPAuth.oauth_delegate) @@ -1431,7 +1434,7 @@ async def _run_passthrough_connect( } ) send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = "u1" server = _build_passthrough_mode_server(server_names[0], auth_type) @@ -1554,7 +1557,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_without_token_surface } ) send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = None tp_server = _build_passthrough_mode_server("tp_server", MCPAuth.true_passthrough) @@ -1620,7 +1623,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_dcr_bridge_challenges } ) send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = None bridge_server = _build_passthrough_mode_server("tp_bridge_server", MCPAuth.true_passthrough).model_copy( update={"dcr_bridge": True} @@ -1691,7 +1694,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_with_token_skips_prob } ) send = AsyncMock() - user_auth = MagicMock() + user_auth = UserAPIKeyAuth() user_auth.user_id = None tp_server = _build_passthrough_mode_server("tp_server", MCPAuth.true_passthrough) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 0573fc0d144..759014b54c5 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -810,6 +810,12 @@ class TestTestConnection: from litellm.proxy._types import LitellmUserRoles from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy.management_endpoints import mcp_management_endpoints + + manager = MCPServerManager() + monkeypatch.setattr(rest_endpoints, "global_mcp_server_manager", manager) + monkeypatch.setattr(mcp_management_endpoints, "global_mcp_server_manager", manager) captured = self._capture_execute(monkeypatch) saved = MCPServer( server_id="saved-server-id", @@ -1313,6 +1319,7 @@ class TestListToolsRestAPI: async def fake_reload(user_id, *, requires_fresh_policy=False): assert user_id == "grant-user" + assert requires_fresh_policy is False return admitted_auth monkeypatch.setattr( @@ -1480,9 +1487,12 @@ class TestListToolsRestAPI: from mcp.types import Tool as MCPTool import litellm.experimental_mcp_client.client as mcp_client_module + from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager from litellm.proxy._experimental.mcp_server.server import MCPServer from litellm.types.mcp import MCPTransport + monkeypatch.setattr(rest_endpoints, "global_mcp_server_manager", MCPServerManager()) + async def fake_contexts(user_api_key_auth): return [user_api_key_auth] @@ -2414,6 +2424,7 @@ class TestCallToolRestAPI: mock_server = MagicMock() mock_server.server_id = "server-1" + mock_server.name = "Example server" def fake_get_mcp_server_by_id(server_id): return mock_server if server_id == "server-1" else None @@ -2431,6 +2442,11 @@ class TestCallToolRestAPI: raising=False, ) + failure_log = AsyncMock() + execute_tool = AsyncMock() + monkeypatch.setattr(rest_endpoints, "_safe_fire_mcp_tool_call_failure_logging", failure_log) + monkeypatch.setattr(rest_endpoints, "execute_mcp_tool", execute_tool) + request_payload = { "server_id": "server-1", "name": "demo-tool", @@ -2452,6 +2468,16 @@ class TestCallToolRestAPI: assert exc_info.value.detail["error"] == "access_denied" assert "server server-1" in exc_info.value.detail["message"] + execute_tool.assert_not_awaited() + failure_log.assert_awaited_once() + logged_data = failure_log.await_args.args[4] + assert logged_data["model"] == "MCP: demo-tool" + assert logged_data["metadata"]["model_group"] == "MCP: demo-tool" + logging_obj = failure_log.await_args.args[0] + assert logging_obj.model_call_details["mcp_tool_call_metadata"] == { + "name": "demo-tool", "mcp_server_name": "Example server", + } + async def test_executes_tool_when_allowed(self, monkeypatch): async def fake_contexts(user_api_key_auth): return [user_api_key_auth] diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py index 4b8d28e2406..a1a022fdd35 100644 --- a/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py +++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_agent_permission_handler.py @@ -1043,3 +1043,119 @@ async def test_managed_target_preserves_ordinary_actor_ceilings_after_key_reload assert await AgentRequestHandler.is_agent_allowed("target", auth) is (permitted and ceiling != "group-without-grant") database.get_data.assert_awaited_once() assert auth.agent_caller == (AgentCaller(team_id="caller-team") if ceiling == "caller-team" else None) + + +@pytest.mark.parametrize( + "direct,teams,selected,explicit,expected", + [ + (False, ("a",), "b", True, "denied"), + (False, ("a",), "a", True, "a"), + (False, ("a",), None, False, "a"), + (False, ("a",), "default-team", False, "a"), + (False, ("a", "b"), "b", True, "b"), + (False, ("b", "a"), None, False, "a"), + (False, ("b", "a"), "default-team", False, "a"), + (False, (), None, False, "denied"), + (True, (), None, False, None), + (True, ("a",), "b", True, "b"), + ], +) +async def test_delegated_team_selection_preserves_the_grant_source( + monkeypatch: pytest.MonkeyPatch, + direct: bool, + teams: tuple[str, ...], + selected: str | None, + explicit: bool, + expected: str | None, +) -> None: + from fastapi import HTTPException + + from litellm.proxy.agent_endpoints.auth import agent_permission_handler as permissions + + sources: Final = [ + (None, frozenset({"actor"}) if direct else frozenset()), + *((team, frozenset({"actor"})) for team in teams), + ] + monkeypatch.setattr(permissions, "_verified_human_agent_sources", AsyncMock(return_value=sources)) + if expected == "denied": + with pytest.raises(HTTPException) as error: + await permissions.resolve_delegated_agent_team("human", "actor", selected, explicit_team=explicit) + assert error.value.status_code == 403 + else: + assert ( + await permissions.resolve_delegated_agent_team("human", "actor", selected, explicit_team=explicit) + == expected + ) + + +@pytest.mark.parametrize( + "team_id,expected", [(None, {"direct"}), ("a", {"direct", "a-only"}), ("b", {"direct", "b-only"})] +) +async def test_delegated_target_grants_do_not_borrow_another_teams_authority( + monkeypatch: pytest.MonkeyPatch, team_id: str | None, expected: set[str] +) -> None: + from litellm.proxy.agent_endpoints.auth import agent_permission_handler as permissions + from litellm.types.agents import AgentResponse + from litellm.types.proxy.agent_identity import ManagedAgentContext + + sources: Final = [(None, frozenset({"direct"})), ("a", frozenset({"a-only"})), ("b", frozenset({"b-only"}))] + monkeypatch.setattr(permissions, "_verified_human_agent_sources", AsyncMock(return_value=sources)) + auth: Final = UserAPIKeyAuth(agent_id="actor", team_id=team_id) + auth.managed_agent_context = ManagedAgentContext(agent_id="actor", mode="delegated", user_id="human") + auth.managed_agent_policy = AgentResponse( + agent_id="actor", + agent_name="Actor", + agent_card_params={}, + object_permission={"object_permission_id": "own", "agents": ["direct", "a-only", "b-only"]}, + ) + assert await AgentRequestHandler.resolve_agent_access(auth) == RestrictedAgentAccess(frozenset(expected)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "managed,enabled,grant,outage,allowed", + [ + (True, True, False, False, False), + (True, True, True, False, True), + (True, False, True, False, False), + (False, True, False, False, True), + (True, True, False, True, False), + ], +) +async def test_target_authorization_uses_live_policy_despite_stale_unmanaged_registry( + monkeypatch: pytest.MonkeyPatch, managed: bool, enabled: bool, grant: bool, outage: bool, allowed: bool +) -> None: + from unittest.mock import MagicMock + + from fastapi import HTTPException + + from litellm.proxy import proxy_server + from litellm.proxy.agent_endpoints import agent_registry + from litellm.types.agents import AgentResponse + from litellm.types.proxy.agent_identity import AgentIdentityBinding + + stale: Final = AgentResponse(agent_id="target", agent_name="Target", agent_card_params={}) + registry: Final = AgentRegistry() + registry.register_agent(stale) + monkeypatch.setattr(agent_registry, "global_agent_registry", registry) + binding: Final = AgentIdentityBinding( + agent_id="target", provider="microsoft_entra", tenant_id="tenant", client_id="client", + issuer="issuer", revision="current", + ) + current: Final = stale.model_copy(update={ + "identity_managed": managed, "identity": binding if managed else None, "enabled": enabled, + }) + database: Final = MagicMock() + database.writer_db.litellm_agentstable.find_unique = AsyncMock( + return_value=current, side_effect=ConnectionError("writer unavailable") if outage else None, + ) + monkeypatch.setattr(proxy_server, "prisma_client", database) + permission: Final = LiteLLM_ObjectPermissionTable(object_permission_id="grant", agents=["target"]) + auth: Final = UserAPIKeyAuth(object_permission=permission if grant else None) + + if outage: + with pytest.raises(HTTPException) as denied: + await AgentRequestHandler.is_agent_allowed("target", auth) + assert denied.value.status_code == 503 + return + assert await AgentRequestHandler.is_agent_allowed("target", auth) is allowed diff --git a/tests/test_litellm/proxy/agent_endpoints/auth/test_managed_authorization.py b/tests/test_litellm/proxy/agent_endpoints/auth/test_managed_authorization.py index 7747e5eff71..eee985f0aca 100644 --- a/tests/test_litellm/proxy/agent_endpoints/auth/test_managed_authorization.py +++ b/tests/test_litellm/proxy/agent_endpoints/auth/test_managed_authorization.py @@ -1,13 +1,15 @@ +from collections.abc import Mapping from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth from litellm.proxy.agent_endpoints.auth.managed_authorization import ( actor_admission_failure, admit_managed_actor, + invocation_target, ) from litellm.proxy.agent_endpoints.identity_store import AgentIdentityStore from litellm.types.agents import AgentResponse @@ -68,6 +70,72 @@ def test_stale_binding_and_unverified_delegation_cannot_pass_admission(context: assert isinstance(actor_admission_failure(agent(), context), AgentIdentityFailure) +def test_caller_cannot_construct_trusted_subject_or_policy() -> None: + context: Final = ManagedAgentContext( + agent_id="agent", binding_revision="current", mode="delegated", user_id="human" + ) + auth: Final = UserAPIKeyAuth.model_validate( + { + "managed_agent_context": context, + "requires_fresh_policy": True, + "authenticated_by_custom_auth": True, + "mcp_explicit_grants_only": True, + "managed_agent_policy": agent(), + "billing_agent_policy": agent(), + "invoked_agent_id": "forged-target", + "agent_invocation_cost": 0.0, + } + ) + assert auth.requires_fresh_policy is False + assert auth.authenticated_by_custom_auth is False + assert "authenticated_by_custom_auth" not in auth.model_dump() + assert auth.mcp_explicit_grants_only is False + assert "mcp_explicit_grants_only" not in auth.model_dump() + assert auth.managed_agent_context is None + assert auth.managed_agent_policy is None + assert auth.billing_agent_policy is None + assert auth.invoked_agent_id is None + assert auth.agent_invocation_cost is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("autonomous", (True, False)) +async def test_invocation_prepares_target_fee_for_the_correct_agent( + monkeypatch: pytest.MonkeyPatch, + autonomous: bool, +) -> None: + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.agent_endpoints import agent_registry + from litellm.proxy.agent_endpoints.auth.managed_authorization import prepare_agent_invocation + from litellm.proxy.agent_endpoints.identity_store import AgentIdentityStore + + target: Final = agent(litellm_params={"cost_per_query": 0.25}) + registry: Final = agent_registry.AgentRegistry() + registry.register_agent(target) + monkeypatch.setattr(agent_registry, "global_agent_registry", registry) + database: Final = MagicMock() + database.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=target) + monkeypatch.setattr(proxy_server, "prisma_client", database) + permission: Final = LiteLLM_ObjectPermissionTable(object_permission_id="invoke-grant", agents=["agent"]) + auth: Final = UserAPIKeyAuth( + agent_id="caller" if autonomous else None, + user_id=None if autonomous else "human", + object_permission=permission, + ) + if autonomous: + caller: Final = agent(agent_id="caller", object_permission=permission.model_dump()) + auth.managed_agent_policy = caller + auth.billing_agent_policy = caller + await prepare_agent_invocation(auth, "agent", AgentIdentityStore.from_client(database)) + assert auth.agent_invocation_cost == pytest.approx(0.25) + assert auth.invoked_agent_id == "agent" + assert auth.billing_agent_policy is not None + assert auth.billing_agent_policy.agent_id == ("caller" if autonomous else "agent") + + @pytest.mark.asyncio async def test_deleted_agent_key_cannot_fall_back_to_unmanaged_authentication() -> None: database: Final = MagicMock() @@ -92,6 +160,24 @@ async def test_agent_history_outage_does_not_permit_legacy_fallback() -> None: assert failure.value.status_code == 503 +@pytest.mark.parametrize( + "route,body,expected", + [ + ("/a2a/agent", {}, "agent"), + ("/a2a/expensive", {"model": "a2a/cheap"}, "expensive"), + ("/a2a/expensive/message/send", {"model": "a2a/cheap"}, "expensive"), + ("/v1/a2a/expensive/message/send", {"model": "a2a/cheap"}, "expensive"), + ("/v1/a2a/agent/", {}, "agent"), + ("/v1/chat/completions", {"model": "a2a/Readable name"}, "Readable name"), + ("/v1/chat/completions", {"model": "a2a/"}, None), + ("/v1/chat/completions", {"model": "ordinary-model"}, None), + ("/a2a", {}, None), + ], +) +def test_invocation_routes_resolve_the_same_target(route: str, body: dict[str, object], expected: str | None) -> None: + assert invocation_target(route, body) == expected + + @pytest.mark.asyncio async def test_agent_admission_database_outage_fails_closed() -> None: database: Final = MagicMock() @@ -162,6 +248,36 @@ def test_execution_mode_must_match_verified_token_mode() -> None: assert "execution mode" in failure.message +@pytest.mark.asyncio +@pytest.mark.parametrize("state,status", [("missing", 403), ("outage", 503), ("denied", 403), ("invalid-fee", 503)]) +async def test_invocation_cannot_bypass_missing_policy_permission_or_invalid_price( + monkeypatch: pytest.MonkeyPatch, state: str, status: int +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import LiteLLM_ObjectPermissionTable + from litellm.proxy.agent_endpoints import agent_registry + from litellm.proxy.agent_endpoints.auth.managed_authorization import prepare_agent_invocation + + registered: Final = agent(litellm_params={"cost_per_query": -1 if state == "invalid-fee" else 0.25}) + registry: Final = agent_registry.AgentRegistry() + registry.register_agent(registered) + monkeypatch.setattr(agent_registry, "global_agent_registry", registry) + database: Final = MagicMock() + database.writer_db.litellm_agentstable.find_unique = AsyncMock( + return_value=None if state == "missing" else registered, + side_effect=RuntimeError("unavailable") if state == "outage" else None, + ) + monkeypatch.setattr(proxy_server, "prisma_client", database) + permission: Final = LiteLLM_ObjectPermissionTable( + object_permission_id="grant", agents=[] if state == "denied" else ["agent"] + ) + auth: Final = UserAPIKeyAuth(user_id="human", object_permission=permission) + with pytest.raises(HTTPException) as failure: + await prepare_agent_invocation(auth, "agent", AgentIdentityStore.from_client(database)) + assert failure.value.status_code == status + assert auth.agent_invocation_cost is None + + @pytest.mark.asyncio async def test_legacy_jwt_cannot_adopt_an_agent_bound_on_another_worker() -> None: database: Final = MagicMock() @@ -192,6 +308,21 @@ async def test_managed_context_or_binding_requires_database(monkeypatch: pytest. assert denied.value.status_code == 503 +@pytest.mark.asyncio +@pytest.mark.parametrize("managed_flag", [False, True]) +async def test_managed_invocation_requires_database(monkeypatch: pytest.MonkeyPatch, managed_flag: bool) -> None: + from litellm.proxy.agent_endpoints import agent_registry + from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry + from litellm.proxy.agent_endpoints.auth.managed_authorization import prepare_agent_invocation + + registry: Final = AgentRegistry() + registry.register_agent(agent(identity_managed=managed_flag)) + monkeypatch.setattr(agent_registry, "global_agent_registry", registry) + with pytest.raises(HTTPException) as denied: + await prepare_agent_invocation(UserAPIKeyAuth(user_id="human"), "agent", None) + assert denied.value.status_code == 503 + + @pytest.mark.asyncio async def test_autonomous_app_rejects_persisted_virtual_key_impersonation() -> None: policy: Final = agent(execution_mode="autonomous") @@ -205,6 +336,121 @@ async def test_autonomous_app_rejects_persisted_virtual_key_impersonation() -> N assert auth.billing_agent_policy is None +@pytest.mark.asyncio +async def test_unknown_invocation_target_leaves_billing_unset(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.agent_endpoints import agent_registry + from litellm.proxy.agent_endpoints.auth.managed_authorization import prepare_agent_invocation + + monkeypatch.setattr(agent_registry, "global_agent_registry", agent_registry.AgentRegistry()) + monkeypatch.setattr(proxy_server, "prisma_client", None) + auth: Final = UserAPIKeyAuth(user_id="human") + await prepare_agent_invocation(auth, "missing", None) + assert auth.invoked_agent_id is None + assert auth.billing_agent_policy is None + + +@pytest.mark.parametrize( + "route,method,allowed", + [ + ("/v1/agents", "GET", True), + ("/v1/agents", "POST", False), + ("/v1/chat/completions", "POST", True), + ("/v1/chat/completions", "DELETE", False), + ("/openai/deployments/model/chat/completions", "POST", True), + ("/engines/openai/model/chat/completions", "POST", True), + ("/openai/deployments/openai/model/images/generations", "POST", True), + ("/openai/deployments/openai/model/images/edits", "POST", True), + ("/v1beta/models/gemini-model:generateContent", "POST", True), + ("/v1/realtime", "GET", True), + ("/v1/realtime", "POST", False), + ("/v1/realtime/client_secrets", "POST", False), + ("/mcp/tools/call", "POST", True), + ("/a2a/target/message/send", "POST", True), + ("/v1/a2a/target/message/send", "POST", True), + ("/v1/videos", "POST", False), + ("/v1/videos/other-video", "GET", False), + ("/v1/search", "POST", False), + ("/search", "POST", False), + ("/v1/agents/target", "PATCH", False), + ("/v1/responses/other-response", "GET", False), + ("/v1/files", "GET", False), + ("/v1/files", "POST", False), + ("/openai/v1/files", "GET", False), + ("/anthropic/v1/files", "GET", False), + ], +) +def test_managed_route_scope_excludes_provider_resources(route: str, method: str, allowed: bool) -> None: + from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_route_allowed + + assert managed_agent_route_allowed(route, method) is allowed + + +@pytest.mark.parametrize( + "route,body,settings,cli_model,path_model,expected", + [ + ("/v1/chat/completions", {"model": "body"}, {"completion_model": "default"}, "cli", "path", "default"), + ("/v1/moderations", {"model": "body"}, {"moderation_model": "default"}, "cli", None, "cli"), + ("/v1/audio/speech", {"model": "body"}, {"completion_model": "ignored"}, None, None, "body"), + ("/openai/deployments/path/embeddings", {"model": "body"}, {}, None, "path", "path"), + ("/v1/messages/count_tokens", {"model": "body"}, {"completion_model": "ignored"}, "cli", None, "body"), + ("/mcp/tools/call", {}, {"completion_model": "ignored"}, "cli", None, None), + ("/v1/images/generations", {"model": "image"}, {"completion_model": "text"}, None, None, "image"), + ("/v1/images/generations", {}, {"image_generation_model": "image"}, None, None, "image"), + ("/v1/images/edits", {}, {"image_generation_model": "image"}, None, None, "image"), + ("/v1/rerank", {"model": "reranker"}, {"completion_model": "text"}, "cli", None, "reranker"), + ("/v1beta/models/path:countTokens", {"model": "body"}, {"completion_model": "text"}, "cli", "path", "path"), + ], +) +def test_managed_inference_resolves_dispatch_precedence( + route: str, + body: Mapping[str, object], + settings: Mapping[str, object], + cli_model: str | None, + path_model: str | None, + expected: str | None, +) -> None: + from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request + + assert managed_inference_request(route, body, settings, cli_model, path_model).get("model") == expected + + +def test_managed_inference_without_any_model_cannot_skip_model_grants(): + from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request + + with pytest.raises(HTTPException, match="explicit or configured model"): + managed_inference_request("/v1/moderations", {}, {}, None) + + +@pytest.mark.parametrize("route", ["/v1/chat/completions", "/v1/images/generations", "/v1/images/edits"]) +def test_managed_inference_query_model_takes_precedence_over_body(route: str): + from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request + + assert managed_inference_request(route, {"model": "body"}, {}, None, query_model="query")["model"] == "query" + + +def test_managed_inference_ignores_unsupported_query_model(): + from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request + + assert ( + managed_inference_request("/v1/messages", {"model": "body"}, {}, None, query_model="query")["model"] == "body" + ) + + +@pytest.mark.parametrize("route", ["/realtime", "/v1/realtime", "/openai/v1/realtime"]) +def test_managed_realtime_requires_a_model_and_ignores_completion_defaults(route: str) -> None: + from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request + + with pytest.raises(HTTPException, match="explicit or configured model"): + managed_inference_request(route, {}, {"completion_model": "allowed-default"}, "cli") + assert ( + managed_inference_request(route, {"model": "requested"}, {"completion_model": "allowed-default"}, "cli")[ + "model" + ] + == "requested" + ) + + @pytest.mark.parametrize("mode,user", [("autonomous", None), ("delegated", "verified-human")]) def test_matching_identity_revision_and_execution_mode_pass_admission(mode: str, user: str | None) -> None: context: Final = ManagedAgentContext.model_validate( @@ -213,6 +459,28 @@ def test_matching_identity_revision_and_execution_mode_pass_admission(mode: str, assert actor_admission_failure(agent(), context) is None +@pytest.mark.asyncio +async def test_unmanaged_agent_invocation_retains_legacy_behavior(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.agent_endpoints import agent_registry + from litellm.proxy.agent_endpoints.auth.managed_authorization import prepare_agent_invocation + + legacy: Final = agent(identity=None, identity_managed=False) + registry: Final = agent_registry.AgentRegistry() + registry.register_agent(legacy) + monkeypatch.setattr(agent_registry, "global_agent_registry", registry) + monkeypatch.setattr(proxy_server, "prisma_client", None) + auth: Final = UserAPIKeyAuth(agent_id="agent") + await admit_managed_actor(auth, None) + database: Final = MagicMock() + database.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=legacy) + await admit_managed_actor(auth, AgentIdentityStore.from_client(database)) + await prepare_agent_invocation(auth, "agent", AgentIdentityStore.from_client(database)) + assert auth.managed_agent_policy is None + assert auth.billing_agent_policy is None + assert auth.invoked_agent_id is None + + @pytest.mark.asyncio async def test_bound_autonomous_actor_is_admitted_without_a_human() -> None: database: Final = MagicMock() @@ -234,6 +502,8 @@ async def test_admitted_managed_actor_requires_fresh_policy_so_revocations_bind_ auth: Final = UserAPIKeyAuth(agent_id="agent") auth.managed_agent_context = ManagedAgentContext(agent_id="agent", binding_revision="current", mode="autonomous") assert auth.requires_fresh_policy is False + assert auth.authenticated_by_custom_auth is False + assert "authenticated_by_custom_auth" not in auth.model_dump() await admit_managed_actor(auth, AgentIdentityStore.from_client(database)) assert auth.requires_fresh_policy is True @@ -283,3 +553,27 @@ async def test_ordinary_agent_admission_preserves_legacy_authentication( assert auth.agent_id == "agent" assert auth.managed_agent_policy is None assert auth.requires_fresh_policy is False + assert auth.authenticated_by_custom_auth is False + assert "authenticated_by_custom_auth" not in auth.model_dump() + + +@pytest.mark.parametrize( + "route", + tuple(dict.fromkeys( + LiteLLMRoutes.openai_routes.value + + LiteLLMRoutes.anthropic_routes.value + + LiteLLMRoutes.google_routes.value + )), +) +def test_registered_inference_routes_have_an_explicit_managed_access_decision(route: str) -> None: + from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_route_allowed + + normalized: Final = route.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1") + unsupported: Final = normalized.startswith(( + "/videos", "/batches", "/files", "/fine_tuning", "/assistants", "/threads", "/utils/", + "/vector_stores", "/vector_store/", "/search", "/containers", "/skills", "/claude-code/", + "/interactions", "/agents", "/responses/{", "/responses/input_tokens", + "/realtime/client_secrets", "/realtime/calls", "/realtime/transcription_sessions", + )) or normalized in ("/models", "/cursor/models", "/cursor/v1/models") + concrete: Final = route.split("?")[0].replace("{model}", "model").replace("{model_name:path}", "model") + assert managed_agent_route_allowed(concrete, None) is not unsupported, route diff --git a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py index 8a7ab0f0001..a5d0d0a3ecc 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py @@ -59,6 +59,7 @@ async def test_invoke_agent_a2a_adds_litellm_data(): # Mock agent mock_agent = MagicMock() + mock_agent.agent_id = "test-agent" mock_agent.agent_card_params = { "url": "http://backend-agent:10001", "name": "Test Agent", @@ -72,6 +73,7 @@ async def test_invoke_agent_a2a_adds_litellm_data(): "jsonrpc": "2.0", "id": "test-id", "method": "message/send", + "metadata": {"model_info": {"id": "caller-supplied-id"}}, "params": { "message": { "role": "user", @@ -153,7 +155,7 @@ async def test_invoke_agent_a2a_adds_litellm_data(): "litellm.a2a_protocol.asend_message", new_callable=AsyncMock, return_value=mock_response, - ), + ) as mock_send_message, patch( "litellm.proxy.proxy_server.general_settings", {}, @@ -190,6 +192,9 @@ async def test_invoke_agent_a2a_adds_litellm_data(): mock_add_data.assert_called_once() # Verify model and custom_llm_provider were set + assert mock_send_message.await_args.kwargs["model"] == "a2a_agent/Test Agent" + assert captured_data["metadata"]["model_group"] == "a2a_agent/Test Agent" + assert captured_data["metadata"]["model_info"] == {"id": mock_agent.agent_id} assert captured_data.get("model") == "a2a_agent/Test Agent" assert captured_data.get("custom_llm_provider") == "a2a_agent" diff --git a/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py index 526f24c5221..bd43cb7ce13 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py @@ -350,6 +350,7 @@ class TestAgentByIdKeyRedaction: test_client = _make_app_with_role(role) with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_prisma.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( return_value=None ) @@ -412,6 +413,7 @@ class TestAgentRBACInternalUser: return_value=_sample_agent_response() ) with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_prisma.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_agentstable.find_unique = AsyncMock( return_value=None ) @@ -1342,6 +1344,7 @@ def test_get_agent_redacts_kill_switch_secret_for_admins_and_hides_it_from_other def _get_as(role: LitellmUserRoles): with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + mock_prisma.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_agentstable.find_unique = AsyncMock(return_value=None) mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) return _make_app_with_role(role).get("/v1/agents/agent-123", headers={"Authorization": "Bearer k"}) diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index b1622e0dff0..640b3d8053d 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -2,15 +2,14 @@ import asyncio import re import time from collections.abc import Mapping, Sequence -from typing import Final, Optional +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch -from fastapi import HTTPException import httpx import pytest +from fastapi import HTTPException -import litellm - +from litellm.caching.dual_cache import DualCache from litellm.proxy._types import ( DEFAULT_JWKS_STALE_TTL, JWTLiteLLMRoleMap, @@ -26,7 +25,6 @@ from litellm.proxy._types import ( RoleBasedPermissions, ScopeMapping, ) -from litellm.caching.dual_cache import DualCache from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry from litellm.proxy.auth.auth_checks import TeamNotFoundError from litellm.proxy.auth.handle_jwt import ( @@ -1637,7 +1635,6 @@ async def test_auth_builder_returns_team_membership_object(): @pytest.mark.asyncio async def test_auth_builder_with_oidc_userinfo_enabled(): """Test that auth_builder uses OIDC UserInfo endpoint when enabled""" - from unittest.mock import MagicMock from litellm.caching import DualCache from litellm.proxy.utils import ProxyLogging @@ -1648,9 +1645,7 @@ async def test_auth_builder_with_oidc_userinfo_enabled(): general_settings = {"enforce_rbac": False} route = "/chat/completions" - user_object = LiteLLM_UserTable( - user_id="test_user_1", user_role=LitellmUserRoles.INTERNAL_USER - ) + user_object = LiteLLM_UserTable(user_id="test_user_1", user_role=LitellmUserRoles.INTERNAL_USER) # Create JWT handler with OIDC UserInfo enabled jwt_handler = JWTHandler() @@ -1677,18 +1672,12 @@ async def test_auth_builder_with_oidc_userinfo_enabled(): # Mock all the dependencies with ( - patch.object( - jwt_handler, "get_oidc_userinfo", new_callable=AsyncMock - ) as mock_get_userinfo, + patch.object(jwt_handler, "get_oidc_userinfo", new_callable=AsyncMock) as mock_get_userinfo, patch.object(jwt_handler, "auth_jwt", new_callable=AsyncMock) as mock_auth_jwt, - patch.object( - JWTAuthManager, "check_rbac_role", new_callable=AsyncMock - ) as mock_check_rbac, + patch.object(JWTAuthManager, "check_rbac_role", new_callable=AsyncMock) as mock_check_rbac, patch.object(jwt_handler, "get_rbac_role", return_value=None) as mock_get_rbac, patch.object(jwt_handler, "get_scopes", return_value=[]) as mock_get_scopes, - patch.object( - jwt_handler, "get_object_id", return_value=None - ) as mock_get_object_id, + patch.object(jwt_handler, "get_object_id", return_value=None) as mock_get_object_id, patch.object( JWTAuthManager, "get_user_info", @@ -1696,9 +1685,7 @@ async def test_auth_builder_with_oidc_userinfo_enabled(): return_value=("test_user_1", "test@example.com", True), ) as mock_get_user_info, patch.object(jwt_handler, "get_org_id", return_value=None) as mock_get_org_id, - patch.object( - jwt_handler, "get_end_user_id", return_value=None - ) as mock_get_end_user_id, + patch.object(jwt_handler, "get_end_user_id", return_value=None) as mock_get_end_user_id, patch.object( JWTAuthManager, "check_admin_access", @@ -1711,9 +1698,7 @@ async def test_auth_builder_with_oidc_userinfo_enabled(): new_callable=AsyncMock, return_value=(None, None), ) as mock_find_team, - patch.object( - JWTAuthManager, "get_all_team_ids", return_value=set() - ) as mock_get_all_team_ids, + patch.object(JWTAuthManager, "get_all_team_ids", return_value=set()) as mock_get_all_team_ids, patch.object( JWTAuthManager, "find_team_with_model_access", @@ -1726,15 +1711,9 @@ async def test_auth_builder_with_oidc_userinfo_enabled(): new_callable=AsyncMock, return_value=(user_object, None, None, None, user_object.user_id), ) as mock_get_objects, - patch.object( - JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock - ) as mock_map_user, - patch.object( - JWTAuthManager, "validate_object_id", return_value=True - ) as mock_validate_object, - patch.object( - JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock - ) as mock_sync_user, + patch.object(JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock) as mock_map_user, + patch.object(JWTAuthManager, "validate_object_id", return_value=True) as mock_validate_object, + patch.object(JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock) as mock_sync_user, ): # Set up mock return values mock_get_userinfo.return_value = userinfo_response @@ -1764,7 +1743,6 @@ async def test_auth_builder_with_oidc_userinfo_enabled(): @pytest.mark.asyncio async def test_auth_builder_with_oidc_userinfo_disabled(): """Test that auth_builder uses JWT validation when OIDC UserInfo is disabled""" - from unittest.mock import MagicMock from litellm.caching import DualCache from litellm.proxy.utils import ProxyLogging @@ -1775,9 +1753,7 @@ async def test_auth_builder_with_oidc_userinfo_disabled(): general_settings = {"enforce_rbac": False} route = "/chat/completions" - user_object = LiteLLM_UserTable( - user_id="test_user_1", user_role=LitellmUserRoles.INTERNAL_USER - ) + user_object = LiteLLM_UserTable(user_id="test_user_1", user_role=LitellmUserRoles.INTERNAL_USER) # Create JWT handler with OIDC UserInfo disabled jwt_handler = JWTHandler() @@ -1801,18 +1777,12 @@ async def test_auth_builder_with_oidc_userinfo_disabled(): # Mock all the dependencies with ( - patch.object( - jwt_handler, "get_oidc_userinfo", new_callable=AsyncMock - ) as mock_get_userinfo, + patch.object(jwt_handler, "get_oidc_userinfo", new_callable=AsyncMock) as mock_get_userinfo, patch.object(jwt_handler, "auth_jwt", new_callable=AsyncMock) as mock_auth_jwt, - patch.object( - JWTAuthManager, "check_rbac_role", new_callable=AsyncMock - ) as mock_check_rbac, + patch.object(JWTAuthManager, "check_rbac_role", new_callable=AsyncMock) as mock_check_rbac, patch.object(jwt_handler, "get_rbac_role", return_value=None) as mock_get_rbac, patch.object(jwt_handler, "get_scopes", return_value=[]) as mock_get_scopes, - patch.object( - jwt_handler, "get_object_id", return_value=None - ) as mock_get_object_id, + patch.object(jwt_handler, "get_object_id", return_value=None) as mock_get_object_id, patch.object( JWTAuthManager, "get_user_info", @@ -1820,9 +1790,7 @@ async def test_auth_builder_with_oidc_userinfo_disabled(): return_value=("test_user_1", None, None), ) as mock_get_user_info, patch.object(jwt_handler, "get_org_id", return_value=None) as mock_get_org_id, - patch.object( - jwt_handler, "get_end_user_id", return_value=None - ) as mock_get_end_user_id, + patch.object(jwt_handler, "get_end_user_id", return_value=None) as mock_get_end_user_id, patch.object( JWTAuthManager, "check_admin_access", @@ -1835,9 +1803,7 @@ async def test_auth_builder_with_oidc_userinfo_disabled(): new_callable=AsyncMock, return_value=(None, None), ) as mock_find_team, - patch.object( - JWTAuthManager, "get_all_team_ids", return_value=set() - ) as mock_get_all_team_ids, + patch.object(JWTAuthManager, "get_all_team_ids", return_value=set()) as mock_get_all_team_ids, patch.object( JWTAuthManager, "find_team_with_model_access", @@ -1850,15 +1816,9 @@ async def test_auth_builder_with_oidc_userinfo_disabled(): new_callable=AsyncMock, return_value=(user_object, None, None, None, user_object.user_id), ) as mock_get_objects, - patch.object( - JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock - ) as mock_map_user, - patch.object( - JWTAuthManager, "validate_object_id", return_value=True - ) as mock_validate_object, - patch.object( - JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock - ) as mock_sync_user, + patch.object(JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock) as mock_map_user, + patch.object(JWTAuthManager, "validate_object_id", return_value=True) as mock_validate_object, + patch.object(JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock) as mock_sync_user, ): # Set up mock return values mock_auth_jwt.return_value = jwt_response @@ -2631,7 +2591,6 @@ async def test_find_and_validate_specific_team_id_with_team_alias(): """ Test that find_and_validate_specific_team_id resolves team by name when team_id is not found """ - from unittest.mock import MagicMock from litellm.caching import DualCache from litellm.proxy._types import LiteLLM_JWTAuth, LiteLLM_TeamTable @@ -2654,9 +2613,7 @@ async def test_find_and_validate_specific_team_id_with_team_alias(): # Mock team object returned by get_team_object_by_alias team_object = LiteLLM_TeamTable(team_id="resolved-team-id", team_alias="my-team") - with patch( - "litellm.proxy.auth.handle_jwt.get_team_object_by_alias", new_callable=AsyncMock - ) as mock_get_by_alias: + with patch("litellm.proxy.auth.handle_jwt.get_team_object_by_alias", new_callable=AsyncMock) as mock_get_by_alias: mock_get_by_alias.return_value = team_object team_id, result_team = await JWTAuthManager.find_and_validate_specific_team_id( @@ -2685,7 +2642,6 @@ async def test_find_and_validate_team_id_takes_precedence_over_name(): """ Test that team_id_jwt_field takes precedence over team_alias_jwt_field """ - from unittest.mock import MagicMock from litellm.caching import DualCache from litellm.proxy._types import LiteLLM_JWTAuth, LiteLLM_TeamTable @@ -2699,9 +2655,7 @@ async def test_find_and_validate_team_id_takes_precedence_over_name(): jwt_handler.update_environment( prisma_client=None, user_api_key_cache=user_api_key_cache, - litellm_jwtauth=LiteLLM_JWTAuth( - team_id_jwt_field="team_id", team_alias_jwt_field="team_alias" - ), + litellm_jwtauth=LiteLLM_JWTAuth(team_id_jwt_field="team_id", team_alias_jwt_field="team_alias"), ) # Token with both team_id and team name @@ -2711,9 +2665,7 @@ async def test_find_and_validate_team_id_takes_precedence_over_name(): team_object = LiteLLM_TeamTable(team_id="direct-team-id") with ( - patch( - "litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock - ) as mock_get_by_id, + patch("litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock) as mock_get_by_id, patch( "litellm.proxy.auth.handle_jwt.get_team_object_by_alias", new_callable=AsyncMock, @@ -2890,7 +2842,6 @@ async def test_get_objects_resolves_org_by_name(): @pytest.mark.asyncio async def test_resolve_jwks_url_passthrough_for_direct_jwks_url(): """Non-discovery URLs are returned unchanged.""" - from unittest.mock import AsyncMock, MagicMock from litellm.caching.dual_cache import DualCache @@ -3143,7 +3094,7 @@ async def test_find_and_validate_specific_team_id_no_hint_for_valid_field(): When team_id_jwt_field is a normal field name (no dot-notation) the error message should not contain a spurious bracket-notation hint. """ - from unittest.mock import AsyncMock, MagicMock + from unittest.mock import MagicMock from litellm.caching.dual_cache import DualCache @@ -3230,8 +3181,8 @@ async def test_find_and_validate_specific_team_id_no_hint_for_valid_field(): async def test_auth_builder_single_team_db_fallback_when_jwt_has_no_team( user_id: str, user_teams: list, - get_team_object_return: Optional[str], - expected_team_id: Optional[str], + get_team_object_return: str | None, + expected_team_id: str | None, expect_get_team_called: bool, expect_get_membership_called: bool, ) -> None: @@ -3244,9 +3195,7 @@ async def test_auth_builder_single_team_db_fallback_when_jwt_has_no_team( if len(user_teams) == 1 and get_team_object_return == "resolved_row": only = user_teams[0] team_table = LiteLLM_TeamTable(team_id=only) - membership = LiteLLM_TeamMembership( - user_id=user_id, team_id=only, litellm_budget_table=None - ) + membership = LiteLLM_TeamMembership(user_id=user_id, team_id=only, litellm_budget_table=None) get_team_return_value = team_table membership_return_value = membership else: @@ -3305,9 +3254,7 @@ async def test_auth_builder_single_team_db_fallback_when_jwt_has_no_team( ), patch.object(JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock), patch.object(JWTAuthManager, "validate_object_id", return_value=True), - patch.object( - JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock - ), + patch.object(JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock), patch( "litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock, @@ -3324,9 +3271,7 @@ async def test_auth_builder_single_team_db_fallback_when_jwt_has_no_team( code = 404 if get_team_object_return == "http_404" else 500 mock_get_team.side_effect = HTTPException( status_code=code, - detail={ - "error": f"Team doesn't exist in db. Team={user_teams[0]}. Create team via `/team/new` call." - }, + detail={"error": f"Team doesn't exist in db. Team={user_teams[0]}. Create team via `/team/new` call."}, ) else: mock_get_team.return_value = get_team_return_value @@ -4047,7 +3992,7 @@ def _encode_rsa_jwt( issuer: str, audience: str, kid: str, - extra_claims: Optional[dict] = None, + extra_claims: dict | None = None, ) -> str: import time @@ -4743,12 +4688,9 @@ async def test_get_objects_team_membership_uses_rebound_user_id(): async def fake_get_team_membership(user_id, team_id, *args, **kwargs): captured["user_id"] = user_id captured["team_id"] = team_id - return None jwt_handler = JWTHandler() - jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( - user_id_jwt_field="email", user_id_upsert=True - ) + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(user_id_jwt_field="email", user_id_upsert=True) with ( patch( @@ -5389,7 +5331,7 @@ async def test_find_team_with_model_access_defers_no_team_403_under_db_fallback( assert team_object is None -def _db_fallback_handler(litellm_jwtauth: Optional[LiteLLM_JWTAuth] = None) -> JWTHandler: +def _db_fallback_handler(litellm_jwtauth: LiteLLM_JWTAuth | None = None) -> JWTHandler: handler = JWTHandler() handler.litellm_jwtauth = litellm_jwtauth or LiteLLM_JWTAuth() return handler @@ -5447,9 +5389,7 @@ async def test_resolve_db_team_fallback_skips_unresolvable_membership(): "expect_403", ), [ - pytest.param( - True, ["team_solo"], None, "team_solo", False, id="flag_on_single_db_team" - ), + pytest.param(True, ["team_solo"], None, "team_solo", False, id="flag_on_single_db_team"), pytest.param( True, ["team_a", "team_b"], @@ -5497,8 +5437,8 @@ async def test_resolve_db_team_fallback_skips_unresolvable_membership(): async def test_auth_builder_db_team_fallback_when_jwt_has_no_team( fallback_to_db_teams: bool, user_teams: list, - header_team_id: Optional[str], - expected_team_id: Optional[str], + header_team_id: str | None, + expected_team_id: str | None, expect_403: bool, ) -> None: """End-to-end auth_builder behavior with no JWT team claims. @@ -5527,9 +5467,7 @@ async def test_auth_builder_db_team_fallback_when_jwt_has_no_team( async def call_auth_builder(): with ( - patch.object( - jwt_handler, "auth_jwt", new_callable=AsyncMock - ) as mock_auth_jwt, + patch.object(jwt_handler, "auth_jwt", new_callable=AsyncMock) as mock_auth_jwt, patch.object(JWTAuthManager, "check_rbac_role", new_callable=AsyncMock), patch.object(jwt_handler, "get_rbac_role", return_value=None), patch.object(jwt_handler, "get_scopes", return_value=[]), @@ -5569,9 +5507,7 @@ async def test_auth_builder_db_team_fallback_when_jwt_has_no_team( ), patch.object(JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock), patch.object(JWTAuthManager, "validate_object_id", return_value=True), - patch.object( - JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock - ), + patch.object(JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock), patch( "litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock, @@ -6765,7 +6701,7 @@ async def test_auth_builder_provisional_header_team_is_not_upserted(): team_id_upsert=True, ) - upsert_by_team: dict[str, Optional[bool]] = {} + upsert_by_team: dict[str, bool | None] = {} async def spy_get_team(team_id, **kwargs): upsert_by_team[team_id] = kwargs.get("team_id_upsert") @@ -6800,9 +6736,7 @@ async def test_auth_builder_provisional_header_team_is_not_upserted(): ), patch.object(JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock), patch.object(JWTAuthManager, "validate_object_id", return_value=True), - patch.object( - JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock - ), + patch.object(JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock), patch( "litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock, @@ -7806,6 +7740,58 @@ async def test_admin_jwt_team_header_only_provisions_during_admission(monkeypatc assert result["team_id"] is None +def _explicit_identity_registry() -> AgentRegistry: + registry: Final = AgentRegistry() + registry.register_agent(AgentResponse( + agent_id="explicit-agent-id", + agent_name="Readable agent name", + agent_card_params={}, + litellm_params={"identity": { + "provider": "microsoft_entra", + "tenant_id": "11111111-1111-4111-8111-111111111111", + "client_id": "22222222-2222-4222-8222-222222222222", + }}, + )) + return registry + + +@pytest.mark.parametrize("claim_field", ["azp", None]) +def test_runtime_json_cannot_establish_a_managed_identity(claim_field: str | None) -> None: + registry: Final = _explicit_identity_registry() + handler: Final = _entra_agent_jwt_handler(claim_field) + claims: Final = { + "iss": "https://login.microsoftonline.com/11111111-1111-4111-8111-111111111111/v2.0", + "tid": "11111111-1111-4111-8111-111111111111", + "azp": "22222222-2222-4222-8222-222222222222", + } + if claim_field is None: + assert JWTAuthManager.resolve_agent_id(handler, claims, registry) is None + else: + with pytest.raises(HTTPException) as failure: + JWTAuthManager.resolve_agent_id(handler, claims, registry) + assert failure.value.status_code == 403 + + +@pytest.mark.parametrize("override", [ + {"iss": "https://attacker.example"}, + {"tid": "33333333-3333-4333-8333-333333333333"}, + {"azp": "33333333-3333-4333-8333-333333333333"}, + {"azp": "explicit-agent-id"}, + {"azp": "Readable agent name"}, +]) +def test_explicit_entra_identity_cannot_be_claimed_via_legacy_lookup(override: Mapping[str, object]) -> None: + registry: Final = _explicit_identity_registry() + handler: Final = _entra_agent_jwt_handler("azp") + with pytest.raises(HTTPException) as failure: + JWTAuthManager.resolve_agent_id(handler, { + "iss": "https://login.microsoftonline.com/11111111-1111-4111-8111-111111111111/v2.0", + "tid": "11111111-1111-4111-8111-111111111111", + "azp": "22222222-2222-4222-8222-222222222222", + **override, + }, registry) + assert failure.value.status_code == 403 + + @pytest.mark.asyncio @pytest.mark.parametrize("existing_user", [False, True]) @pytest.mark.parametrize("warm_cache", [False, True]) @@ -7853,3 +7839,389 @@ async def test_scope_admin_admission_resolves_existing_user_without_provisioning users.create.assert_not_awaited() if existing_user: assert users.find_unique.await_count == (0 if warm_cache else 1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["autonomous", "both", "delegated"]) +@pytest.mark.parametrize("audience_validation", (True, False)) +@pytest.mark.parametrize( + "route,allowed", + [ + ("/chat/completions", True), ("/v1/messages", True), ("/v1/responses", True), + ("/mcp-rest/tools/call", True), ("/a2a/target", True), + ("/v1/files", False), ("/v1/batches", False), ("/v1/vector_stores", False), + ("/v1/containers", False), ("/openai/v1/files", False), + ("/v1/responses/other-response", False), ("/v1/realtime/client_secrets", False), + ], +) +async def test_managed_application_uses_persisted_identity_without_provisioning_human( + monkeypatch: pytest.MonkeyPatch, mode: str, audience_validation: bool, route: str, allowed: bool +) -> None: + from litellm.types.proxy.agent_identity import AgentIdentityBinding + + tenant: Final = "11111111-1111-4111-8111-111111111111" + client_id: Final = "22222222-2222-4222-8222-222222222222" + principal: Final = "33333333-3333-4333-8333-333333333333" + issuer: Final = f"https://login.microsoftonline.com/{tenant}/v2.0" + jwks_url: Final = "https://login.microsoftonline.test/managed-keys" + monkeypatch.setenv("JWT_PUBLIC_KEY_URL", jwks_url) + monkeypatch.setenv("JWT_ISSUER", issuer) + monkeypatch.setenv("JWT_AUDIENCE", "api://gateway") + private_key, jwk = _get_rsa_key_and_jwk(kid="managed-key") + cache: Final = DualCache() + cache.set_cache(key=f"litellm_jwt_auth_keys_{jwks_url}", value=[jwk]) + handler: Final = JWTHandler() + handler.update_environment(None, cache, LiteLLM_JWTAuth(user_id_upsert=True)) + binding: Final = AgentIdentityBinding( + agent_id="stable-id", + provider="microsoft_entra", + issuer=issuer, + tenant_id=tenant, + client_id=client_id, + service_principal_id=principal, + revision="revision-one", + required_roles=("Agent.Invoke",), + ) + agent: Final = AgentResponse.model_validate( + { + "agent_id": "stable-id", + "agent_name": "A readable name", + "agent_card_params": {}, + "identity": binding, + "identity_managed": True, + "execution_mode": mode, + } + ) + database: Final = MagicMock() + database.writer_db.litellm_agentidentity.find_unique = AsyncMock(return_value=binding) + database.writer_db.litellm_agentidentity.update_many = AsyncMock(return_value=1) + database.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=agent) + database.writer_db.litellm_verifiedsubject.find_unique = AsyncMock(return_value=None) + database.db.litellm_usertable.upsert = AsyncMock() + token: Final = _encode_rsa_jwt( + private_key, + issuer=issuer, + audience="api://gateway", + kid="managed-key", + extra_claims={ + "tid": tenant, + "azp": client_id, + "oid": principal, + "roles": ["Agent.Invoke"], + "idtyp": "app", + }, + ) + arguments: Final = dict( + api_key=token, + jwt_handler=handler, + request_data={}, + general_settings={}, + route=route, + prisma_client=database, + user_api_key_cache=cache, + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + if not audience_validation: + monkeypatch.delenv("JWT_AUDIENCE") + if mode == "delegated" or not audience_validation or not allowed: + with pytest.raises(HTTPException) as failure: + await JWTAuthManager.auth_builder(**arguments) + assert failure.value.status_code == 403 + else: + result: Final = await JWTAuthManager.auth_builder(**arguments) + auth: Final = JWTAuthManager.user_api_key_auth_from_result(result) + assert auth.agent_id == "stable-id" + assert auth.api_key is None + assert auth.token is None + assert auth.user_id is None + assert auth.team_id is None + assert auth.managed_agent_context is not None + assert auth.managed_agent_context.mode == "autonomous" + assert result["is_proxy_admin"] is False + database.db.litellm_usertable.upsert.assert_not_awaited() + + +@pytest.mark.parametrize("claim_value", ["managed", "Readable managed agent"]) +def test_legacy_claim_cannot_select_a_top_level_entra_binding(claim_value: str) -> None: + from litellm.types.proxy.agent_identity import AgentIdentityBinding + + registry: Final = AgentRegistry() + registry.register_agent( + AgentResponse( + agent_id="managed", + agent_name="Readable managed agent", + agent_card_params={}, + identity_managed=True, + identity=AgentIdentityBinding( + agent_id="managed", + provider="microsoft_entra", + tenant_id="tenant", + client_id="client", + service_principal_id="principal", + issuer="issuer", + revision="revision", + ), + ) + ) + with pytest.raises(HTTPException) as denied: + JWTAuthManager.resolve_agent_id(_entra_agent_jwt_handler("agent"), {"agent": claim_value}, registry) + assert denied.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["human", "config-agent", "managed-agent"]) +async def test_database_free_jwt_admission_with_entra_shaped_claims(monkeypatch: pytest.MonkeyPatch, kind: str) -> None: + issuer: Final = "https://login.microsoftonline.com/test-tenant/v2.0" + jwks_url: Final = "https://login.microsoftonline.test/config-only-keys" + monkeypatch.setenv("JWT_PUBLIC_KEY_URL", jwks_url) + monkeypatch.setenv("JWT_ISSUER", issuer) + monkeypatch.setenv("JWT_AUDIENCE", "api://gateway") + private_key, jwk = _get_rsa_key_and_jwk(kid="config-key") + cache: Final = DualCache() + cache.set_cache(key=f"litellm_jwt_auth_keys_{jwks_url}", value=[jwk]) + registry: Final = AgentRegistry() + registry.register_agent( + AgentResponse( + agent_id="configured", + agent_name="Configured", + agent_card_params={}, + identity_managed=kind == "managed-agent", + ) + ) + handler: Final = JWTHandler() + handler.update_environment(None, cache, LiteLLM_JWTAuth(agent_id_jwt_field="agent", admin_allowed_routes=["llm_api_routes"])) + handler.bind_agent_lookup(registry) + token: Final = _encode_rsa_jwt( + private_key, + issuer=issuer, + audience="api://gateway", + kid="config-key", + extra_claims={ + "tid": "test-tenant", + "azp": "application", + "scope": "litellm_proxy_admin", + **({"agent": "configured"} if kind != "human" else {}), + }, + ) + arguments: Final = dict( + api_key=token, + jwt_handler=handler, + request_data={}, + general_settings={}, + route="/chat/completions", + prisma_client=None, + user_api_key_cache=cache, + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + if kind == "managed-agent": + with pytest.raises(HTTPException) as denied: + await JWTAuthManager.auth_builder(**arguments) + assert denied.value.status_code == 403 + else: + result: Final = await JWTAuthManager.auth_builder(**arguments) + auth: Final = JWTAuthManager.user_api_key_auth_from_result(result) + assert auth.agent_id == ("configured" if kind == "config-agent" else None) + assert auth.managed_agent_context is None + assert result["is_proxy_admin"] is True + + +@pytest.mark.parametrize( + "issuer,audience,disabled,expected", + [ + (None, "gateway", False, False), + ("trusted", "gateway", False, True), + ("trusted", None, True, False), + ("other", "gateway", False, False), + ], +) +def test_managed_issuer_requires_configured_audience_validation( + monkeypatch: pytest.MonkeyPatch, issuer: str | None, audience: str | None, disabled: bool, expected: bool +) -> None: + from litellm.proxy._types import JWTIssuerConfig + + monkeypatch.delenv("JWT_ISSUER", raising=False) + monkeypatch.delenv("JWT_AUDIENCE", raising=False) + handler: Final = JWTHandler() + handler.update_environment( + None, + DualCache(), + LiteLLM_JWTAuth( + issuers=[ + JWTIssuerConfig(issuer="trusted", audience=audience, disable_audience_validation=disabled), + ] + ), + ) + assert handler.managed_issuer_is_trusted(issuer) is expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("authentication_write", ["success", "revoked", "unavailable"]) +async def test_managed_jwt_reuses_binding_lookup_but_rechecks_disabled_policy( + monkeypatch: pytest.MonkeyPatch, authentication_write: str +) -> None: + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.types.proxy.agent_identity import AgentIdentityBinding + + issuer: Final = "https://login.microsoftonline.com/tenant/v2.0" + jwks_url: Final = "https://identity.example/managed-jwks" + private_key, jwk = _get_rsa_key_and_jwk("managed-cache") + cache: Final = UserApiKeyCache() + cache.set_cache(f"litellm_jwt_auth_keys_{jwks_url}", [jwk]) + monkeypatch.setenv("JWT_PUBLIC_KEY_URL", jwks_url) + monkeypatch.setenv("JWT_ISSUER", issuer) + monkeypatch.setenv("JWT_AUDIENCE", "gateway") + binding: Final = AgentIdentityBinding( + agent_id="managed", provider="microsoft_entra", issuer=issuer, tenant_id="tenant", + client_id="client", service_principal_id="principal", revision="current", + ) + agent: Final = AgentResponse( + agent_id="managed", agent_name="Managed", agent_card_params={}, identity_managed=True, identity=binding, + ) + database: Final = MagicMock() + database.writer_db.litellm_agentidentity.find_unique = AsyncMock(return_value=binding) + database.writer_db.litellm_agentidentity.update_many = AsyncMock(return_value=1) + database.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=agent) + handler: Final = JWTHandler() + handler.update_environment(database, cache, LiteLLM_JWTAuth()) + token: Final = _encode_rsa_jwt( + private_key, issuer, "gateway", "managed-cache", {"tid": "tenant", "azp": "client", "oid": "principal"} + ) + arguments: Final = dict( + api_key=token, jwt_handler=handler, request_data={}, general_settings={}, route="/chat/completions", + prisma_client=database, user_api_key_cache=cache, parent_otel_span=None, proxy_logging_obj=MagicMock(), + ) + for _ in range(2): + result: Final = await JWTAuthManager.authorize_jwt(**arguments) + assert result["agent_id"] == "managed" + database.writer_db.litellm_agentidentity.find_unique.assert_awaited_once() + assert database.writer_db.litellm_agentstable.find_unique.await_count == 2 + assert database.writer_db.litellm_agentidentity.update_many.await_count == 2 + if authentication_write != "success": + database.writer_db.litellm_agentidentity.update_many.return_value = 0 + database.writer_db.litellm_agentidentity.update_many.side_effect = ( + RuntimeError("storage unavailable") if authentication_write == "unavailable" else None + ) + with pytest.raises(HTTPException) as failed_write: + await JWTAuthManager.authorize_jwt(**arguments) + assert failed_write.value.status_code == (503 if authentication_write == "unavailable" else 403) + assert database.writer_db.litellm_agentidentity.update_many.await_count == 3 + return + database.writer_db.litellm_agentstable.find_unique.return_value = agent.model_copy(update={"enabled": False}) + with pytest.raises(HTTPException) as denied: + await JWTAuthManager.authorize_jwt(**arguments) + assert denied.value.status_code == 403 + assert database.writer_db.litellm_agentidentity.update_many.await_count == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "team_route_allowed,team_claim,db_fallback", + [ + (True, None, False), + (False, None, False), + (True, "other-team", False), + (True, "granting-team", False), + (True, "other-team", True), + (True, "alias:other-team", False), + (True, "alias:other-team", True), + ], +) +async def test_delegated_jwt_uses_granting_team_policy_before_route_authorization( + monkeypatch: pytest.MonkeyPatch, team_route_allowed: bool, team_claim: str | None, db_fallback: bool +) -> None: + from litellm.proxy.agent_endpoints.auth import agent_permission_handler + from litellm.proxy.auth import handle_jwt + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.types.proxy.agent_identity import ManagedAgentContext + + issuer: Final = "https://login.microsoftonline.com/11111111-1111-4111-8111-111111111111/v2.0" + jwks_url: Final = "https://identity.example/delegated-jwks" + private_key, jwk = _get_rsa_key_and_jwk("delegated-team") + cache: Final = UserApiKeyCache() + cache.set_cache(f"litellm_jwt_auth_keys_{jwks_url}", [jwk]) + monkeypatch.setenv("JWT_PUBLIC_KEY_URL", jwks_url) + monkeypatch.setenv("JWT_ISSUER", issuer) + monkeypatch.setenv("JWT_AUDIENCE", "gateway") + database: Final = MagicMock() + database.writer_db.litellm_agentidentity.update_many = AsyncMock(return_value=1) + handler: Final = JWTHandler() + handler.update_environment( + database, + cache, + LiteLLM_JWTAuth( + team_allowed_routes=["/chat/completions" if team_route_allowed else "/embeddings"], + team_id_jwt_field="team" if team_claim is not None else None, + team_alias_jwt_field="team_alias" if team_claim is not None else None, + fallback_to_db_teams=db_fallback, + ), + ) + context: Final = ManagedAgentContext( + agent_id="delegated-agent", binding_revision="revision", mode="delegated", user_id="human" + ) + monkeypatch.setattr(handle_jwt, "resolve_managed_agent", AsyncMock(return_value=context)) + monkeypatch.setattr( + agent_permission_handler, + "_verified_human_agent_sources", + AsyncMock(return_value=(("granting-team", frozenset(("delegated-agent",))),)), + ) + team: Final = LiteLLM_TeamTable(team_id="granting-team", models=["allowed-model"], max_budget=5) + + async def team_policy(team_id: str, **kwargs: object) -> LiteLLM_TeamTable: + return team if team_id == team.team_id else LiteLLM_TeamTable(team_id=team_id) + + load_team: Final = AsyncMock(side_effect=team_policy) + monkeypatch.setattr(handle_jwt, "get_team_object", load_team) + monkeypatch.setattr( + handle_jwt, "get_team_object_by_alias", AsyncMock(return_value=LiteLLM_TeamTable(team_id="other-team")) + ) + monkeypatch.setattr( + handle_jwt, + "get_user_object", + AsyncMock(return_value=LiteLLM_UserTable(user_id="human", teams=["granting-team", "other-team"])), + ) + monkeypatch.setattr(handle_jwt, "get_team_membership", AsyncMock(return_value=None)) + token: Final = _encode_rsa_jwt( + private_key, + issuer, + "gateway", + "delegated-team", + { + "sub": "human", + **( + {"team_alias": "other-team"} + if team_claim == "alias:other-team" + else {"team": team_claim} + if team_claim + else {} + ), + }, + ) + pending: Final = JWTAuthManager.authorize_jwt( + api_key=token, + jwt_handler=handler, + request_data={"model": "allowed-model"}, + general_settings={}, + route="/chat/completions", + request_method="POST", + prisma_client=database, + user_api_key_cache=cache, + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + if not team_route_allowed or (team_claim in ("other-team", "alias:other-team") and not db_fallback): + with pytest.raises(HTTPException) as failure: + await pending + assert failure.value.status_code == 403 + if team_claim is None: + assert "granting team" in failure.value.detail + load_team.assert_not_awaited() + return + result: Final = await pending + assert result["team_id"] == "granting-team" + assert result["team_object"] == team + assert result["user_id"] == "human" + assert result["managed_agent_context"] == context + if team_claim != "granting-team": + assert any(call.kwargs.get("check_db_only") is True for call in load_team.call_args_list) diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index b2da7f30926..ef6832ef77b 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -9278,6 +9278,8 @@ async def test_websocket_auth_hands_the_reservation_to_the_socket_state(): ) async def auth_that_reserves(request, api_key): + assert request.method == "GET" + assert request.query_params.get("model") == "gpt-realtime" request.state.budget_reservation = reservation return UserAPIKeyAuth(token="hashed", budget_reservation=reservation) @@ -9383,6 +9385,209 @@ def test_identity_prefetch_keys_match_what_auth_reads_for_the_request(): ) +@pytest.mark.asyncio +@pytest.mark.parametrize("invoke", [False, True]) +async def test_centralized_authorization_preserves_database_free_config_agents(monkeypatch, invoke: bool): + from litellm.proxy import proxy_server + from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry + from litellm.proxy.agent_endpoints import agent_registry + from litellm.proxy.auth.user_api_key_auth import _authorize_authenticated_request + + for name, value in { + **_proxy_attrs_for_centralized_checks(), + "prisma_client": None, + "proxy_logging_obj": MagicMock(post_call_failure_hook=AsyncMock(return_value=None)), + }.items(): + monkeypatch.setattr(proxy_server, name, value) + registry = AgentRegistry() + registry.load_agents_from_config( + [{"agent_name": "config-agent", "agent_card_params": {"name": "Config", "url": "http://localhost:9999"}}] + ) + monkeypatch.setattr(agent_registry, "global_agent_registry", registry) + registered = registry.get_agent_by_name("config-agent") + model = "a2a/config-agent" if invoke else "test-model" + auth = UserAPIKeyAuth(agent_id=registered.agent_id, jwt_claims={"agent": "config-agent"}, models=[model]) + data = {"model": model, "messages": [{"role": "user", "content": "hi"}]} + assert ( + await _authorize_authenticated_request( + auth, _alias_request("/v1/chat/completions", data), data, "/v1/chat/completions", "jwt-token" + ) + is None + ) + assert auth.managed_agent_policy is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("verified_identity", [False, True]) +async def test_managed_actor_cannot_access_provider_resource_routes(monkeypatch, verified_identity: bool): + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import _authorize_authenticated_request + from litellm.types.agents import AgentResponse + from litellm.types.proxy.agent_identity import AgentIdentityBinding + + policy = AgentResponse( + agent_id="managed", + agent_name="Managed", + agent_card_params={}, + identity_managed=True, + identity=AgentIdentityBinding( + agent_id="managed", + provider="microsoft_entra", + tenant_id="tenant", + client_id="application", + service_principal_id="principal", + issuer="issuer", + revision="revision", + ), + object_permission={"models": ["test-model"]}, + ) + database = MagicMock() + database.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=policy) + for name, value in { + **_proxy_attrs_for_centralized_checks(), + "prisma_client": database, + "proxy_logging_obj": MagicMock(post_call_failure_hook=AsyncMock(return_value=None)), + }.items(): + monkeypatch.setattr(proxy_server, name, value) + request = _alias_request("/v1/files", {}) + request.scope["method"] = "GET" + from litellm.types.proxy.agent_identity import ManagedAgentContext + + auth = UserAPIKeyAuth(agent_id="managed", api_key="persisted-key", models=["test-model"]) + if verified_identity: + auth.managed_agent_context = ManagedAgentContext( + agent_id="managed", binding_revision="revision", mode="autonomous" + ) + with pytest.raises(ProxyException) as denied: + await _authorize_authenticated_request(auth, request, {}, "/v1/files", "persisted-key") + assert denied.value.code == "403" + if verified_identity: + assert denied.value.message == "Agent identities can only access inference and agent discovery routes" + else: + assert denied.value.message == "This agent requires its bound identity provider token" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("requested", [None, "test-model"]) +@pytest.mark.parametrize("grant_default", [False, True]) +@pytest.mark.parametrize( + "route,settings,cli_model", + [ + ("/v1/chat/completions", {"completion_model": "forbidden-model"}, None), + ("/v1/responses", {"completion_model": "forbidden-model"}, None), + ("/v1/messages", {"completion_model": "forbidden-model"}, None), + ("/v1/moderations", {"moderation_model": "forbidden-model"}, None), + ("/v1/audio/transcriptions", {"moderation_model": "forbidden-model"}, None), + ("/v1/audio/speech", {}, "forbidden-model"), + ("/v1/chat/completions", {}, "forbidden-model"), + ("/v1/images/generations", {"image_generation_model": "forbidden-model"}, None), + ("/v1/images/edits", {"image_generation_model": "forbidden-model"}, None), + ], +) +async def test_managed_agent_cannot_bypass_grants_with_server_default( + monkeypatch, requested, route, settings, cli_model, grant_default +): + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import _authorize_authenticated_request + from litellm.types.agents import AgentResponse + from litellm.types.proxy.agent_identity import AgentIdentityBinding, ManagedAgentContext + + policy = AgentResponse( + agent_id="managed", + agent_name="Managed", + agent_card_params={}, + identity_managed=True, + identity=AgentIdentityBinding( + agent_id="managed", + provider="microsoft_entra", + tenant_id="tenant", + client_id="application", + service_principal_id="principal", + issuer="issuer", + revision="revision", + ), + object_permission={"models": ["test-model", "forbidden-model"] if grant_default else ["test-model"]}, + ) + database = MagicMock() + database.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=policy) + for name, value in { + **_proxy_attrs_for_centralized_checks(), + "prisma_client": database, + "general_settings": settings, + "user_model": cli_model, + "proxy_logging_obj": MagicMock(post_call_failure_hook=AsyncMock(return_value=None)), + }.items(): + monkeypatch.setattr(proxy_server, name, value) + data = {"messages": [{"role": "user", "content": "hi"}], **({"model": requested} if requested else {})} + auth = UserAPIKeyAuth(agent_id="managed") + auth.managed_agent_context = ManagedAgentContext( + agent_id="managed", binding_revision="revision", mode="autonomous" + ) + if not grant_default: + with pytest.raises(ProxyException) as denied: + await _authorize_authenticated_request(auth, _alias_request(route, data), data, route, "persisted-key") + assert denied.value.code == "403" + assert "forbidden-model" in denied.value.message + return + with patch( + "litellm.proxy.spend_tracking.budget_reservation.reserve_budget_for_request", + new_callable=AsyncMock, + ) as reserve: + reserve.return_value = None + assert ( + await _authorize_authenticated_request(auth, _alias_request(route, data), data, route, "persisted-key") + is None + ) + reserve.assert_awaited_once() + assert reserve.call_args.kwargs["request_body"]["model"] == "forbidden-model" + + +@pytest.mark.asyncio +async def test_managed_jwt_cannot_be_downgraded_into_virtual_key_mapping(monkeypatch: pytest.MonkeyPatch) -> None: + from typing import Final + + from litellm.proxy import proxy_server + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.types.agents import AgentResponse + from litellm.types.proxy.agent_identity import AgentIdentityBinding + + binding: Final = AgentIdentityBinding( + agent_id="managed", provider="microsoft_entra", issuer="issuer", tenant_id="tenant", + client_id="client", service_principal_id="principal", revision="current", + ) + agent: Final = AgentResponse( + agent_id="managed", agent_name="Managed", agent_card_params={}, + identity_managed=True, identity=binding, execution_mode="autonomous", + ) + client: Final = MagicMock() + client.writer_db.litellm_agentidentity.find_unique = AsyncMock(return_value=binding) + client.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=agent) + handler: Final = MagicMock() + handler.is_jwt.return_value = True + handler.litellm_jwtauth = LiteLLM_JWTAuth(virtual_key_claim_field="sub") + handler.auth_jwt = AsyncMock(return_value={ + "iss": "issuer", "tid": "tenant", "azp": "client", "oid": "principal", "sub": "mapped-key", + }) + for name, value in { + **_proxy_attrs_for_centralized_checks(), + "general_settings": {"enable_jwt_auth": True}, "premium_user": True, + "prisma_client": client, "jwt_handler": handler, "user_api_key_cache": UserApiKeyCache(), + "proxy_logging_obj": MagicMock(post_call_failure_hook=AsyncMock(return_value=None)), + }.items(): + monkeypatch.setattr(proxy_server, name, value) + for _ in range(2): + with pytest.raises(ProxyException) as failure: + await _user_api_key_auth_builder( + request=_alias_request("/v1/chat/completions", {}), api_key="Bearer verified.jwt.token", + azure_api_key_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, + azure_apim_header=None, request_data={}, + ) + assert failure.value.code == "403" + assert "without virtual-key mapping" in failure.value.message + client.writer_db.litellm_agentidentity.find_unique.assert_awaited_once() + assert client.writer_db.litellm_agentstable.find_unique.await_count == 2 + + @pytest.mark.asyncio async def test_virtual_key_cannot_enter_checks_as_an_identity_managed_actor(monkeypatch: pytest.MonkeyPatch) -> None: from typing import Final @@ -9410,3 +9615,76 @@ async def test_virtual_key_cannot_enter_checks_as_an_identity_managed_actor(monk UserAPIKeyAuth(agent_id="bound"), request, data, "/v1/chat/completions", "sk-test" ) checks.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enterprise", [False, True]) +@pytest.mark.parametrize("credential", ["custom-credential", "sk-custom-credential"]) +@pytest.mark.parametrize("granted", [False, True]) +async def test_custom_auth_grants_reach_managed_targets_without_a_virtual_key_row( + monkeypatch: pytest.MonkeyPatch, enterprise: bool, credential: str, granted: bool +) -> None: + import importlib + from typing import Final + + from litellm.proxy import proxy_server + from litellm.proxy.agent_endpoints import agent_registry + from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry + from litellm.proxy.agent_endpoints.auth.agent_permission_handler import AgentRequestHandler + from litellm.types.agents import AgentResponse + from litellm.types.proxy.agent_identity import AgentIdentityBinding + + target: Final = AgentResponse( + agent_id="target", agent_name="Target", agent_card_params={}, identity_managed=True, + identity=AgentIdentityBinding( + agent_id="target", provider="microsoft_entra", tenant_id="tenant", client_id="client", + issuer="issuer", revision="current", + ), + ) + registry: Final = AgentRegistry() + registry.register_agent(target) + trusted: Final = UserAPIKeyAuth( + api_key=credential, object_permission={"object_permission_id": "custom", "agents": ["target"] if granted else ["other"]} + ) + custom: Final = AsyncMock(return_value=trusted) + database: Final = MagicMock() + database.get_data = AsyncMock(return_value=None) + database.writer_db.litellm_agentstable.find_unique = AsyncMock(return_value=target) + for name, value in { + **_proxy_server_attrs_for_custom_auth(user_custom_auth=None if enterprise else custom), + "prisma_client": database, + }.items(): + monkeypatch.setattr(proxy_server, name, value) + module: Final = importlib.import_module("litellm.proxy.auth.user_api_key_auth") + monkeypatch.setattr(module, "enterprise_custom_auth", custom if enterprise else None) + monkeypatch.setattr(agent_registry, "global_agent_registry", registry) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", False, raising=False) + admitted: Final = await _user_api_key_auth_builder( + request=_alias_request("/a2a/target/message/send", {}), api_key=f"Bearer {credential}", + azure_api_key_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, + azure_apim_header=None, request_data={}, + ) + assert await AgentRequestHandler.is_agent_allowed("target", admitted) is granted + custom.assert_awaited_once() + database.get_data.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_enterprise_custom_auth_key_return_stays_a_proxy_validated_key(monkeypatch: pytest.MonkeyPatch) -> None: + import importlib + from typing import Final + + from litellm.proxy import proxy_server + + custom: Final = AsyncMock(return_value="sk-master-key") + for name, value in _proxy_server_attrs_for_custom_auth(user_custom_auth=custom).items(): + monkeypatch.setattr(proxy_server, name, value) + module: Final = importlib.import_module("litellm.proxy.auth.user_api_key_auth") + monkeypatch.setattr(module, "enterprise_custom_auth", custom) + admitted: Final = await _user_api_key_auth_builder( + request=_alias_request("/v1/chat/completions", {}), api_key="Bearer external-credential", + azure_api_key_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, + azure_apim_header=None, request_data={}, + ) + assert admitted.authenticated_by_custom_auth is False + assert admitted.via_virtual_key is True diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index 7929a0b21af..6087aa1f570 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -1,6 +1,7 @@ import io import json -from typing import get_type_hints +from collections.abc import Mapping +from typing import Literal, get_type_hints from unittest.mock import AsyncMock, MagicMock, patch import orjson @@ -1210,3 +1211,42 @@ class TestCoerceNumericFormFields: numeric_fields=self.numeric_fields, ) assert result == {"n": 3, "temperature": None, "image": buffer} + + +@pytest.mark.parametrize( + "kind,settings,cli,path,body,expected", + [ + ("completion", {"completion_model": "default"}, "cli", "path", "body", "default"), + ("completion", {}, "cli", "path", "body", "cli"), + ("completion", {}, None, "path", "body", "path"), + ("completion", {}, None, None, "body", "body"), + ( + "image_generation", + {"completion_model": "text", "image_generation_model": "image"}, + None, + None, + "body", + "image", + ), + ("image_generation", {"image_generation_model": "image"}, "cli", "path", "body", "cli"), + ("image_generation", {"image_generation_model": "image"}, None, "path", "body", "path"), + ("image_edit", {"completion_model": "text", "image_generation_model": "image"}, None, None, "body", "text"), + ("image_edit", {"image_generation_model": "image"}, None, "path", "body", "path"), + ("image_edit", {"image_generation_model": "image"}, None, None, "body", "image"), + ("moderation", {"moderation_model": "mod"}, "cli", None, "body", "cli"), + ("speech", {"completion_model": "text"}, None, None, "body", "body"), + ("body", {"completion_model": "text"}, "cli", None, "body", "body"), + ("path", {"completion_model": "text"}, "cli", "path", "body", "path"), + ], +) +def test_shared_inference_model_selection_preserves_handler_precedence( + kind: Literal["completion", "image_generation", "image_edit", "moderation", "speech", "body", "path"], + settings: Mapping[str, object], + cli: str | None, + path: str | None, + body: str, + expected: str, +) -> None: + from litellm.proxy.common_utils.http_parsing_utils import resolve_inference_model + + assert resolve_inference_model(body, settings, cli, path, kind=kind) == expected diff --git a/tests/test_litellm/proxy/common_utils/test_registry_read_through.py b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py index ca2ff8bcce1..9e20386bf3d 100644 --- a/tests/test_litellm/proxy/common_utils/test_registry_read_through.py +++ b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py @@ -177,7 +177,7 @@ async def test_get_agent_with_read_through_recovers_agent_created_on_sibling_rep assert agent.agent_id == agent_id prisma_client.db.litellm_agentstable.find_unique.assert_awaited_once_with( where={"agent_id": agent_id}, - include={"object_permission": True}, + include={"object_permission": True, "identity": True}, ) @@ -202,7 +202,7 @@ async def test_get_agent_with_read_through_recovers_agent_by_name(clean_agent_re assert agent.agent_name == agent_name prisma_client.db.litellm_agentstable.find_unique.assert_awaited_with( where={"agent_name": agent_name}, - include={"object_permission": True}, + include={"object_permission": True, "identity": True}, ) @@ -521,3 +521,33 @@ async def test_resync_agents_waits_for_agent_reload_and_skips_duplicate_registra assert await resync_task is True assert len(clean_agent_registry.agent_list) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("lookup", ["agent-id", "Agent name"]) +async def test_agent_read_through_hydrates_identity_binding(lookup, clean_agent_registry, fresh_agent_read_through, monkeypatch): + from types import SimpleNamespace + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy.common_utils.registry_read_through import get_agent_with_read_through + + binding = { + "agent_id": "agent-id", "provider": "microsoft_entra", "tenant_id": "tenant", "client_id": "client", + "issuer": "https://login.microsoftonline.com/tenant/v2.0", "revision": "revision", + } + + async def load_row(*, where, include): + if where == {"agent_id": "Agent name"}: + return None + row = FakeAgentRow("agent-id", "Agent name").model_dump() + return SimpleNamespace(model_dump=lambda: {**row, "identity": binding if include.get("identity") else None}) + + prisma = MagicMock() + prisma.db.litellm_agentstable.find_unique = AsyncMock(side_effect=load_row) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + agent = await get_agent_with_read_through(lookup) + assert agent is not None + assert agent.identity is not None + assert agent.identity.model_dump(include=set(binding)) == binding + assert clean_agent_registry.get_agent_by_id(agent_id="agent-id").identity == agent.identity diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 3be8501f5da..b546b9eb965 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -7592,3 +7592,117 @@ async def test_success_tpm_accounting_keeps_the_admission_target_after_an_alias_ charged: Final = {op["key"]: op["increment_value"] for op in ops} assert charged[admission_bucket] == 150 - stash.reserved_tokens assert not any(":target-b" in key for key in charged) + + +@pytest.mark.parametrize("self_call", [False, True]) +async def test_managed_invocations_enforce_actor_and_target_rate_policies( + monkeypatch: pytest.MonkeyPatch, self_call: bool +) -> None: + from litellm.types.agents import AgentResponse + + actor: Final = AgentResponse( + agent_id="actor", agent_name="Actor", agent_card_params={}, rpm_limit=10, tpm_limit=1000 + ) + target: Final = AgentResponse( + agent_id="target", + agent_name="Target", + agent_card_params={}, + rpm_limit=1, + tpm_limit=1000, + session_rpm_limit=1, + session_tpm_limit=1000, + ) + auth: Final = UserAPIKeyAuth(agent_id="actor") + auth.managed_agent_policy = actor + auth.invoked_agent_id = "actor" if self_call else "target" + auth.invoked_agent_policy = actor if self_call else target + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + monkeypatch.setattr(handler, "_get_agent_from_registry", lambda _: None) + descriptors: Final = handler._create_rate_limit_descriptors( + user_api_key_dict=auth, + data={"model": "a2a/target", "litellm_session_id": "session"}, + rpm_limit_type=None, + tpm_limit_type=None, + model_has_failures=False, + ) + limits: Final = {(item["key"], item["value"]): item["rate_limit"]["requests_per_unit"] for item in descriptors} + assert limits == ( + {("agent", "actor"): 10} + if self_call + else {("agent", "actor"): 10, ("agent", "target"): 1, ("agent_session", "target:session"): 1} + ) + assert len(descriptors) == len(limits) + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 20, + "litellm_session_id": "session", + }, + call_type="acompletion", + ) + stash: Final = get_request_stash() + assert stash is not None and stash.reserved_tokens > 3 + response: Final = ModelResponse(usage=Usage(prompt_tokens=2, completion_tokens=1, total_tokens=3)) + operations: Final = handler._build_success_event_pipeline_operations( + kwargs={"standard_logging_object": {"metadata": {"agent_id": auth.invoked_agent_id, "session_id": "session"}}}, + response_obj=response, + rate_limit_type="total", + ) + increments: Final = {op["key"]: op["increment_value"] for op in operations} + for scope in stash.reserved_scopes: + if scope[0] in ("agent", "agent_session"): + assert increments[handler.create_rate_limit_keys(*scope, "tokens")] == 3 - stash.reserved_tokens + + +@pytest.mark.parametrize("route", ["/a2a/expensive", "/a2a/expensive/message/send", "/v1/a2a/expensive/message/send"]) +async def test_a2a_url_target_owns_invocation_fee_and_request_limit( + monkeypatch: pytest.MonkeyPatch, route: str +) -> None: + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy import proxy_server + from litellm.proxy.agent_endpoints import agent_registry + from litellm.proxy.agent_endpoints.auth.managed_authorization import invocation_target, prepare_agent_invocation + from litellm.proxy.agent_endpoints.identity_store import AgentIdentityStore + from litellm.types.agents import AgentResponse + + expensive: Final = AgentResponse( + agent_id="expensive", agent_name="Expensive", agent_card_params={}, rpm_limit=1, + litellm_params={"cost_per_query": 0.25}, + ) + cheap: Final = AgentResponse( + agent_id="cheap", agent_name="Cheap", agent_card_params={}, rpm_limit=100, + litellm_params={"cost_per_query": 0.01}, + ) + registry: Final = agent_registry.AgentRegistry() + registry.register_agent(expensive) + registry.register_agent(cheap) + monkeypatch.setattr(agent_registry, "global_agent_registry", registry) + database: Final = MagicMock() + database.writer_db.litellm_agentstable.find_unique = AsyncMock( + side_effect=lambda where, include: {"expensive": expensive, "cheap": cheap}[where["agent_id"]] + ) + monkeypatch.setattr(proxy_server, "prisma_client", database) + auth: Final = UserAPIKeyAuth(agent_id="caller") + auth.managed_agent_policy = AgentResponse( + agent_id="caller", agent_name="Caller", agent_card_params={}, + object_permission={"object_permission_id": "both-targets", "agents": ["expensive", "cheap"]}, + ) + body: Final = {"model": "a2a/cheap"} + target: Final = invocation_target(route, body) + assert target is not None + await prepare_agent_invocation(auth, target, AgentIdentityStore.from_client(database)) + assert auth.invoked_agent_id == "expensive" + assert auth.invoked_agent_policy == expensive + assert auth.agent_invocation_cost == pytest.approx(0.25) + cache: Final = DualCache() + limiter: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + await _rpm_request(limiter, cache, auth, "a2a/cheap") + with pytest.raises(HTTPException) as denied: + await _rpm_request(limiter, cache, auth, "a2a/cheap") + assert denied.value.status_code == 429 + assert "expensive" in str(denied.value.detail) diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index a5b2d8b0b8d..84da227c0a6 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -2714,3 +2714,34 @@ async def test_track_cost_callback_failure_alert_never_carries_request_metadata_ assert "headers" in failure_debug_lines[0] else: assert failure_debug_lines == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("identity_field", ["agent_id", "billing_agent_id"]) +async def test_autonomous_llm_callback_persists_without_human_or_key(identity_field: str) -> None: # test-quality-ok: verifies anonymous-agent charges reach the persistence boundary; no injection seam + kwargs: Final = { + "call_type": "acompletion", + "model": "test-model", + "response_cost": 0.01, + "litellm_params": {"metadata": {identity_field: "autonomous-agent"}}, + } + with patch( + "litellm.proxy.hooks.proxy_track_cost_callback._update_database_and_spend_counters", + new_callable=AsyncMock, + return_value=False, + ) as persist: + await _ProxyDBLogger()._PROXY_track_cost_callback( + kwargs=kwargs, completion_response=ModelResponse(), start_time=datetime.now(), end_time=datetime.now() + ) + persist.assert_awaited_once() + assert persist.call_args.kwargs["response_cost"] == 0.01 + assert persist.call_args.kwargs["user_id"] is None + assert persist.call_args.kwargs["user_api_key"] is None + assert persist.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][identity_field] == "autonomous-agent" + + +@pytest.mark.parametrize("agent_id,expected", [(None, False), ("autonomous-agent", True)]) +def test_autonomous_agent_cost_tracking_needs_no_human_or_virtual_key(agent_id: str | None, expected: bool) -> None: + assert _should_track_cost_callback( + user_api_key=None, user_id=None, team_id=None, end_user_id=None, call_type="acompletion", agent_id=agent_id + ) is expected diff --git a/tests/test_litellm/proxy/management_endpoints/sso/test_agent_subject_enrollment.py b/tests/test_litellm/proxy/management_endpoints/sso/test_agent_subject_enrollment.py new file mode 100644 index 00000000000..68fe77cc76a --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/sso/test_agent_subject_enrollment.py @@ -0,0 +1,114 @@ +from types import SimpleNamespace +from typing import Final +from unittest.mock import AsyncMock + +import pytest +from fastapi import HTTPException + +from litellm.proxy.management_endpoints.sso.agent_subject_enrollment import ( + enroll_microsoft_subject, + microsoft_interactive_subject, +) + +TENANT: Final = "11111111-1111-4111-8111-111111111111" +OID: Final = "22222222-2222-4222-8222-222222222222" + + +def test_enrollment_uses_provider_object_id_and_configured_tenant() -> None: + subject: Final = microsoft_interactive_subject( + TENANT, {"id": OID, "mail": "alias@example.com", "tid": "untrusted"}, {} + ) + assert subject is not None + assert subject.oid == OID + assert subject.tenant_id == TENANT + assert subject.issuer == f"https://login.microsoftonline.com/{TENANT}/v2.0" + + +@pytest.mark.parametrize("tenant", [None, "common", "organizations", "invalid"]) +def test_multitenant_sso_does_not_guess_the_subject_tenant(tenant: str | None) -> None: + assert microsoft_interactive_subject(tenant, {"id": OID, "tid": TENANT}, {}) is None + + +@pytest.mark.parametrize("response", [{"mail": "user@example.com"}, {"id": "user@example.com"}, {"id": 42}]) +def test_email_and_configurable_aliases_are_not_human_subject_proof(response: dict[str, object]) -> None: + assert microsoft_interactive_subject(TENANT, response, {}) is None + + +@pytest.mark.parametrize( + "endpoint", ["MICROSOFT_USERINFO_ENDPOINT", "MICROSOFT_TOKEN_ENDPOINT", "MICROSOFT_AUTHORIZATION_ENDPOINT"] +) +def test_custom_provider_endpoints_do_not_enroll_trusted_microsoft_subjects(endpoint: str) -> None: + assert microsoft_interactive_subject(TENANT, {"id": OID}, {endpoint: "https://custom.example"}) is None + + +@pytest.mark.asyncio +async def test_interactive_enrollment_preserves_the_canonical_local_user() -> None: + table: Final = AsyncMock() + table.upsert.return_value = SimpleNamespace(kind="human", user_id="canonical", verified_via="sso_interactive") + client: Final = SimpleNamespace(writer_db=SimpleNamespace(litellm_verifiedsubject=table)) + subject: Final = microsoft_interactive_subject(TENANT, {"id": OID}, {}) + assert subject is not None + await enroll_microsoft_subject(subject, "canonical", client) + table.upsert.assert_awaited_once_with( + where={"issuer_tenant_id_oid": {"issuer": subject.issuer, "tenant_id": TENANT, "oid": OID}}, + data={ + "create": { + "issuer": subject.issuer, + "tenant_id": TENANT, + "oid": OID, + "user_id": "canonical", + "verified_via": "sso_interactive", + }, + "update": {}, + }, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("user_id,verified_via", [("another-user", "sso_interactive"), ("canonical", "untrusted")]) +async def test_interactive_enrollment_does_not_reassign_an_existing_subject(user_id: str, verified_via: str) -> None: + table: Final = AsyncMock() + table.upsert.return_value = SimpleNamespace(kind="human", user_id=user_id, verified_via=verified_via) + client: Final = SimpleNamespace(writer_db=SimpleNamespace(litellm_verifiedsubject=table)) + with pytest.raises(HTTPException) as failure: + await enroll_microsoft_subject(microsoft_interactive_subject(TENANT, {"id": OID}, {}), "canonical", client) + assert failure.value.status_code == 403 + assert table.upsert.call_args.kwargs["data"]["update"] == {} + + +@pytest.mark.asyncio +async def test_enrollment_storage_failure_is_not_a_successful_login() -> None: + table: Final = AsyncMock() + table.upsert.side_effect = RuntimeError("database unavailable") + client: Final = SimpleNamespace(writer_db=SimpleNamespace(litellm_verifiedsubject=table)) + with pytest.raises(HTTPException) as failure: + await enroll_microsoft_subject(microsoft_interactive_subject(TENANT, {"id": OID}, {}), "canonical", client) + assert failure.value.status_code == 503 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("user_id", [None, "", 42]) +async def test_enrollment_requires_a_canonical_local_user(user_id: object) -> None: + table: Final = AsyncMock() + client: Final = SimpleNamespace(writer_db=SimpleNamespace(litellm_verifiedsubject=table)) + await enroll_microsoft_subject(microsoft_interactive_subject(TENANT, {"id": OID}, {}), user_id, client) + table.upsert.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_untrusted_metadata_cannot_enroll_a_human() -> None: + table: Final = AsyncMock() + client: Final = SimpleNamespace(writer_db=SimpleNamespace(litellm_verifiedsubject=table)) + await enroll_microsoft_subject({"issuer": "forged", "tenant_id": TENANT, "oid": OID}, "canonical", client) + table.upsert.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_scim_agent_subject_cannot_be_enrolled_as_a_human() -> None: + table: Final = AsyncMock() + table.upsert.return_value = SimpleNamespace(kind="agent_user", user_id=None, verified_via="scim") + client: Final = SimpleNamespace(writer_db=SimpleNamespace(litellm_verifiedsubject=table)) + with pytest.raises(HTTPException) as failure: + await enroll_microsoft_subject(microsoft_interactive_subject(TENANT, {"id": OID}, {}), "canonical", client) + assert failure.value.status_code == 403 + assert table.upsert.call_args.kwargs["data"]["update"] == {} diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 21c0f565486..9cdf5e9d6ff 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -206,6 +206,7 @@ def test_microsoft_sso_handler_openid_from_response_with_custom_attributes(): def test_get_microsoft_callback_response(): # Arrange mock_request = MagicMock(spec=Request) + mock_request.scope = {} mock_response = { "mail": "microsoft_user@example.com", "displayName": "Microsoft User", @@ -2995,6 +2996,7 @@ class TestCLIKeyRegenerationFlow: from litellm.proxy.management_endpoints.ui_sso import cli_sso_callback mock_request = MagicMock(spec=Request) + mock_request.scope = {} mock_request.base_url = "https://proxy.example.com/" mock_user_info = LiteLLM_UserTable( @@ -3158,6 +3160,7 @@ class TestCLIKeyRegenerationFlow: # Mock request mock_request = MagicMock(spec=Request) + mock_request.scope = {} mock_request.base_url = "http://internal-proxy.local/" # Test data @@ -7106,6 +7109,7 @@ class TestCliSsoAttributionMetadata: from litellm.proxy.management_endpoints.types import CustomOpenID mock_request = MagicMock(spec=Request) + mock_request.scope = {} mock_request.base_url = "http://internal-proxy.local/" session_key = "cli-session-new-user" mock_user_info = LiteLLM_UserTable( @@ -7220,6 +7224,7 @@ class TestCliSsoAttributionMetadata: ) mock_request = MagicMock(spec=Request) + mock_request.scope = {} mock_request.base_url = "http://internal-proxy.local/" session_key = "cli-session-4567890" mock_user_info = LiteLLM_UserTable( @@ -8751,6 +8756,7 @@ async def test_redirect_from_openid_persists_assertion_under_canonical_user_id() assertion = assertion_from_sso_login(_ema_id_token(), "rt_1") assert assertion is not None mock_request = MagicMock(spec=Request) + mock_request.scope = {} mock_request.base_url = "http://localhost:4000/" mock_request.cookies = {} @@ -8822,6 +8828,7 @@ async def test_cli_completion_persists_assertion_under_db_user_id(): assertion = assertion_from_sso_login(_ema_id_token(), None) assert assertion is not None mock_request = MagicMock(spec=Request) + mock_request.scope = {} mock_request.base_url = "http://localhost:4000/" user_info = MagicMock() @@ -8989,6 +8996,7 @@ async def test_browser_funnel_reports_an_uncaptured_assertion(monkeypatch, caplo """Wiring: the browser login path must reach the diagnostic, not just define it.""" monkeypatch.setenv("GOOGLE_CLIENT_ID", "cid") mock_request = MagicMock(spec=Request) + mock_request.scope = {} mock_request.base_url = "http://localhost:4000/" mock_request.cookies = {} @@ -9059,6 +9067,7 @@ async def test_cli_funnel_reports_an_uncaptured_assertion(monkeypatch, caplog): monkeypatch.setenv("MICROSOFT_CLIENT_ID", "cid") mock_request = MagicMock(spec=Request) + mock_request.scope = {} mock_request.base_url = "http://localhost:4000/" user_info = MagicMock() @@ -9134,6 +9143,7 @@ def _cli_callback_kwargs(flow): def _cli_callback_request(): mock_request = MagicMock(spec=Request) + mock_request.scope = {} mock_request.base_url = "http://localhost:4000/" return mock_request @@ -9438,3 +9448,45 @@ class TestSessionTokenCookie: resp = Response() set_session_token_cookie(resp, _make_http_request(), "jwt-token-value") assert "Secure" in self._cookie(resp) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("trusted", [False, True]) +@pytest.mark.parametrize("storage_available", [False, True]) +async def test_cli_sign_in_enrolls_only_verified_subjects_before_completing( + monkeypatch: pytest.MonkeyPatch, trusted: bool, storage_available: bool +) -> None: + from typing import Final + + from litellm.proxy.management_endpoints import ui_sso + from litellm.types.proxy.agent_identity import MicrosoftInteractiveSubject + + flow: Final[dict[str, object]] = {} + kwargs: Final = _cli_callback_kwargs(flow) + subject: Final = MicrosoftInteractiveSubject(issuer="issuer", tenant_id="tenant", oid="subject") + kwargs["request"].scope = {"litellm_microsoft_interactive_subject": subject if trusted else subject.model_dump()} + table: Final = kwargs["prisma_client"].writer_db.litellm_verifiedsubject + table.upsert = AsyncMock( + return_value=SimpleNamespace(kind="human", user_id="cli-user-id", verified_via="sso_interactive"), + side_effect=None if storage_available else RuntimeError("storage unavailable"), + ) + monkeypatch.setattr(ui_sso, "get_user_info_from_db", AsyncMock(return_value=_cli_callback_user_info([]))) + monkeypatch.setattr(ui_sso, "fetch_cli_sso_team_details", AsyncMock(return_value=())) + monkeypatch.setattr(ui_sso, "retain_sso_identity_assertion_for_ema", AsyncMock()) + if trusted and not storage_available: + with pytest.raises(HTTPException) as error: + await ui_sso._complete_cli_sso_callback_session(**kwargs) + assert error.value.status_code == 503 + assert "sso_complete" not in flow + return + response: Final = await ui_sso._complete_cli_sso_callback_session(**kwargs) + assert response.status_code == 200 + assert flow["session_data"]["user_id"] == "cli-user-id" + if trusted: + table.upsert.assert_awaited_once_with( + where={"issuer_tenant_id_oid": {"issuer": "issuer", "tenant_id": "tenant", "oid": "subject"}}, + data={"create": {"issuer": "issuer", "tenant_id": "tenant", "oid": "subject", + "user_id": "cli-user-id", "verified_via": "sso_interactive"}, "update": {}}, + ) + else: + table.upsert.assert_not_awaited() diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 7378564f7a8..0157200ed5c 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -4699,24 +4699,28 @@ def _config_agent(agent_name: str) -> Dict[str, Any]: } -class _FakeAgentRow: - """Stand-in for a prisma agent record: supports dict() and .object_permission.""" +def _agent_db_row(agent_id: str, agent_name: str): + import json + from datetime import datetime, timezone - def __init__(self, agent_id: str, agent_name: str) -> None: - self.agent_id = agent_id - self.agent_name = agent_name - self.object_permission = None - self.spend = 0.0 + from prisma.models import LiteLLM_AgentsTable - def __iter__(self): - return iter( - { - "agent_id": self.agent_id, - "agent_name": self.agent_name, - "agent_card_params": {"name": self.agent_name, "url": "http://db-agent"}, - "litellm_params": {}, - }.items() - ) + return LiteLLM_AgentsTable( + agent_id=agent_id, + agent_name=agent_name, + agent_card_params=json.dumps({"name": agent_name, "url": "http://db-agent"}), + extra_headers=[], + agent_access_groups=[], + access_group_ids=[], + spend=0.0, + identity_managed=False, + enabled=True, + execution_mode="autonomous", + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + created_by="admin", + updated_by="admin", + ) @pytest.mark.asyncio @@ -4740,7 +4744,7 @@ async def test_ProxyConfig__init_agents_in_db_keeps_config_defined_agents(clean_ ) prisma_client = MagicMock() - prisma_client.db.litellm_agentstable.find_many = AsyncMock(return_value=[_FakeAgentRow("db-id", "db-agent")]) + prisma_client.db.litellm_agentstable.find_many = AsyncMock(return_value=[_agent_db_row("db-id", "db-agent")]) await ProxyConfig()._init_agents_in_db(prisma_client=prisma_client) @@ -4777,7 +4781,7 @@ async def test_ProxyStartupEvent_jwt_auth_resolves_agent_claims_against_live_reg elif agents_source == "db": prisma_client = MagicMock() prisma_client.db.litellm_agentstable.find_many = AsyncMock( - return_value=[_FakeAgentRow("db-id", "loaded-agent")] + return_value=[_agent_db_row("db-id", "loaded-agent")] ) await ProxyConfig()._init_agents_in_db(prisma_client=prisma_client) else: diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 3b265653b12..5b3ca27061b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -4,6 +4,7 @@ import datetime import hashlib import json import re +import sqlite3 from datetime import timezone from unittest.mock import AsyncMock, MagicMock, patch @@ -3785,6 +3786,7 @@ class TestSpendLogsPayload: "status": "success", "mcp_namespaced_tool_name": None, "agent_id": None, + "billing_agent_id": None, } ) @@ -6590,9 +6592,7 @@ def test_key_spend_report_scopes_to_caller_key(client, monkeypatch): def test_key_spend_report_scopes_a_cli_session_to_the_per_user_alias_not_the_login_token(client, monkeypatch): - mock_prisma = _spend_report_mock_prisma( - query_raw_returns=[{"api_key": "cli-session-alice", "total_cost": 1.5}] - ) + mock_prisma = _spend_report_mock_prisma(query_raw_returns=[{"api_key": "cli-session-alice", "total_cost": 1.5}]) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( @@ -7142,9 +7142,8 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc rep_call = emitted[2] assert f"DISTINCT ON ({SESSION_GROUP_KEY_SQL})" in rep_call[0] assert ( - f"ORDER BY {SESSION_GROUP_KEY_SQL}, call_type IN ('call_mcp_tool', 'list_mcp_tools'), \"startTime\" DESC" - in rep_call[0] - ), "the session representative must prefer the newest non-MCP call" + f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL + ) in rep_call[0] assert rep_call[-2] == ["sess-1", "req-solo"] assert rep_call[-1] == ["hashed-key", "hashed-key"] finally: @@ -7634,6 +7633,39 @@ def test_ui_view_request_response_internal_user_missing_row_forbidden(client, mo app.dependency_overrides.pop(ps.user_api_key_auth, None) +@pytest.mark.parametrize( + ("parent_status", "child_status", "expected"), + [("failure", "success", "failure"), ("success", "failure", "success")], +) +def test_session_representative_uses_completed_agent_outcome(parent_status, child_status, expected): + with sqlite3.connect(":memory:") as connection: + connection.execute( + 'CREATE TABLE logs (request_id TEXT, call_type TEXT, status TEXT, "startTime" TEXT, "endTime" TEXT)' + ) + connection.executemany( + "INSERT INTO logs VALUES (?, ?, ?, ?, ?)", + ( + ("parent", "asend_message", parent_status, "10:00:00", "10:00:05"), + ("nested-agent", "asend_message", child_status, "10:00:01", "10:00:03"), + ("llm", "acompletion", "success", "10:00:02", "10:00:04"), + ("tool", "call_mcp_tool", child_status, "10:00:04", "10:00:04"), + ), + ) + result = connection.execute( + "SELECT request_id, status FROM logs ORDER BY " + + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL + + " LIMIT 1" + ).fetchone() + assert result == ("parent", expected) + connection.execute("DELETE FROM logs WHERE call_type = 'asend_message'") + fallback = connection.execute( + "SELECT request_id, status FROM logs ORDER BY " + + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL + + " LIMIT 1" + ).fetchone() + assert fallback == ("llm", "success") + + @pytest.mark.asyncio async def test_calculate_spend_unpriced_model_returns_400(): model = "openrouter/unit-test-unpriced-model" diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py b/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py index 54e5a6d5385..6752c91e9f2 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_query_optimization.py @@ -539,7 +539,7 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): async def mock_query_raw(sql_query, *params): if "COUNT(*) AS total_count" in sql_query: return [{"total_count": 60}] - if "DISTINCT ON" in sql_query: + if "AS session_representatives" in sql_query: return representative_rows return session_rows @@ -584,9 +584,11 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): rep_sql = emitted[2][0] assert f"DISTINCT ON ({group_key})" in rep_sql, f"page must return one row per session. SQL was:\n{rep_sql}" - assert f"ORDER BY {group_key}, call_type IN ('call_mcp_tool', 'list_mcp_tools'), \"startTime\" DESC" in rep_sql, ( - "the session representative must prefer the newest non-MCP call" - ) + assert ( + f"ORDER BY {group_key}, (call_type = 'asend_message') DESC, " + "CASE WHEN call_type = 'asend_message' THEN \"endTime\" END DESC NULLS LAST, " + "call_type IN ('call_mcp_tool', 'list_mcp_tools'), \"startTime\" DESC" + ) in rep_sql, "the session representative must prefer the final agent outcome, then the newest non-MCP call" assert "COUNT(*) OVER ()" not in rep_sql assert [row["request_id"] for row in response["data"]] == ["req-1", "req-2"] diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index f3991c0e494..782ce40e624 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -5254,6 +5254,24 @@ def test_spend_log_request_id_is_the_response_id_a_bridged_messages_caller_recei ) +def test_failed_agent_request_keeps_registered_display_name(): + agent_model: Final = "a2a_agent/Research Agent" + payload: Final = get_logging_payload( + kwargs={ + "model": agent_model, + "call_type": "asend_message", + "litellm_params": { + "metadata": {"model_group": agent_model, "model_info": {"id": "registered-agent"}, "status": "failure"} + }, + }, + response_obj=ValueError("Agent action denied"), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + assert payload["model"] == agent_model + assert payload["status"] == "failure" + assert payload["model_id"] == "registered-agent" + _CLI_SESSION_ALIAS: Final = "cli-session-alice" _CLI_SESSION_TOKEN: Final = "cli-session-Qm7xJ2kP9sLw4vT1nR8yAa" @@ -5379,3 +5397,21 @@ def test_baseline_estimate_metadata_comes_from_the_logging_stamp() -> None: assert result["autorouter_savings_estimate"] == recorded absent: Final = _get_spend_logs_metadata({"autorouter_savings_estimate": supplied}) # mutable-ok: legacy metadata helper accepts dicts assert absent["autorouter_savings_estimate"] is None + + +@pytest.mark.parametrize("billing_agent", [None, "authenticated-agent"]) +def test_untrusted_agent_label_cannot_replace_verified_billing_identity(billing_agent: str | None) -> None: + kwargs = { + "model": "gpt-4", + "litellm_params": {"metadata": { + "user_api_key": "test-key", + "agent_id": "header-selected-agent", + "billing_agent_id": billing_agent, + }}, + } + payload = get_logging_payload( + kwargs=kwargs, response_obj={"id": "request"}, + start_time=datetime.datetime.now(timezone.utc), end_time=datetime.datetime.now(timezone.utc), + ) + assert payload["agent_id"] == "header-selected-agent" + assert payload["billing_agent_id"] == billing_agent diff --git a/tests/test_litellm/proxy/test_route_a2a_models.py b/tests/test_litellm/proxy/test_route_a2a_models.py index 35308474949..0429dd97a1c 100644 --- a/tests/test_litellm/proxy/test_route_a2a_models.py +++ b/tests/test_litellm/proxy/test_route_a2a_models.py @@ -149,6 +149,9 @@ async def test_route_a2a_model_read_through_recovers_agent_created_on_sibling_re prisma_client.db.litellm_agentstable.find_unique = AsyncMock( side_effect=[None, _DbAgentRow("a2a-sibling-replica-agent-id", agent_name)] ) + prisma_client.writer_db.litellm_agentstable.find_unique = AsyncMock( + return_value=_DbAgentRow("a2a-sibling-replica-agent-id", agent_name) + ) monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) monkeypatch.setattr(proxy_server, "store_model_in_db", True) From 657bb777fa557f7089c56fedb42f3056d7b55847 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:12:28 -0700 Subject: [PATCH 3/7] fix(hosted_vllm): keep reasoning_content on replayed assistant messages (#43599) * fix(hosted_vllm): keep reasoning_content on assistant messages in _transform_messages vLLM accepts reasoning_content (200 on the wire) and qwen/deepseek/glm chat templates consume it, so popping it made reasoning models lose earlier reasoning across tool loops. thinking_blocks is still removed for vLLM compatibility. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(hosted_vllm): forward replayed reasoning_content only when it is a string * test(integration): cover hosted_vllm reasoning_content replay across endpoints * test(integration): require the surviving worker to serve its held requests in the sigkill chaos cell --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: yassin --- .../llms/hosted_vllm/chat/transformation.py | 7 +- ...pic_messages_hosted_vllm_reasoning_wire.py | 223 +++++++ ...est_hosted_vllm_reasoning_content_chaos.py | 393 ++++++++++++ ...test_hosted_vllm_reasoning_content_wire.py | 565 ++++++++++++++++++ .../test_hosted_vllm_chat_transformation.py | 39 +- 5 files changed, 1220 insertions(+), 7 deletions(-) create mode 100644 tests/integration/messages_endpoint/chat_bridge/test_anthropic_messages_hosted_vllm_reasoning_wire.py create mode 100644 tests/integration/providers/test_hosted_vllm_reasoning_content_chaos.py create mode 100644 tests/integration/providers/test_hosted_vllm_reasoning_content_wire.py diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 32c60bd01b5..43eb2af171e 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -161,13 +161,14 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): """ Support translating: - video files from file_id or file_data to video_url - - thinking_blocks and reasoning_content on assistant messages are removed, - and content lists are converted to strings for vLLM compatibility + - thinking_blocks and non-string reasoning_content on assistant messages + are removed, and content lists are converted to strings for vLLM compatibility """ for message in messages: if message["role"] == "assistant": message.pop("thinking_blocks", None) - message.pop("reasoning_content", None) + if not isinstance(message.get("reasoning_content"), str): + message.pop("reasoning_content", None) existing_content = message.get("content") if isinstance(existing_content, list): text_parts = [] diff --git a/tests/integration/messages_endpoint/chat_bridge/test_anthropic_messages_hosted_vllm_reasoning_wire.py b/tests/integration/messages_endpoint/chat_bridge/test_anthropic_messages_hosted_vllm_reasoning_wire.py new file mode 100644 index 00000000000..2359a8f768f --- /dev/null +++ b/tests/integration/messages_endpoint/chat_bridge/test_anthropic_messages_hosted_vllm_reasoning_wire.py @@ -0,0 +1,223 @@ +import json +import uuid +from typing import Final + +import anthropic +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "glm-reasoning" +_API_KEY: Final = "synthetic-hosted-vllm-key" +_TOOL_USE_ID: Final = "toolu_weather_1" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]]) +_TOOLS: Final[list[dict[str, JsonValue]]] = [ + { + "name": "get_weather", + "description": "Get the current weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } +] + + +def _completion(identity: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "It is raining."}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35}, + } + ).encode() + + +def _streamed_completion(identity: str) -> Reply: + chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": _BACKEND} + frames: Final = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "It is raining."}}]}, + { + **chunk, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35}, + }, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(b"data: " + json.dumps(frame).encode() + b"\n\n" for frame in frames), b"data: [DONE]\n\n"), + ) + + +def _tool_loop(thinking: str, marker: str) -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": f"What is the weather in Paris? {marker}"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": thinking, "signature": "opaque-signature"}, + {"type": "text", "text": "Let me check."}, + {"type": "tool_use", "id": _TOOL_USE_ID, "name": "get_weather", "input": {"city": "Paris"}}, + ], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": _TOOL_USE_ID, "content": "light rain, 14C"}], + }, + ] + + +def _expected_upstream(thinking: str, marker: str) -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": f"What is the weather in Paris? {marker}"}, + { + "role": "assistant", + "content": "Let me check.", + "reasoning_content": thinking, + "tool_calls": [ + { + "id": _TOOL_USE_ID, + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"city": "Paris"})}, + } + ], + }, + {"role": "tool", "tool_call_id": _TOOL_USE_ID, "content": "light rain, 14C"}, + ] + + +def _only_body(wire: Wire) -> dict[str, JsonValue]: + received: Final = wire.drain() + assert [(request.method, request.target) for request in received] == [("POST", "/v1/chat/completions")] + return _JSON_OBJECT.validate_json(received[0].body) + + +def _sent_messages(body: dict[str, JsonValue]) -> list[dict[str, JsonValue]]: + return _MESSAGES.validate_python(body["messages"]) + + +def _spend_status(identity: str) -> JsonValue: + rows: Final = eventually( + lambda: read_rows('SELECT status FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (identity,)), + lambda found: len(found) == 1, + seconds=70, + ) + return rows[0]["status"] + + +def _post_messages(gateway: Gateway, model: str, messages: list[dict[str, JsonValue]]) -> dict[str, JsonValue]: + response: Final = gateway.request( + "POST", + "/v1/messages", + {"model": model, "max_tokens": 256, "messages": messages, "cache": {"no-cache": True}}, + headers={"anthropic-version": "2023-06-01"}, + ) + assert response.status_code == 200, response.text + return _JSON_OBJECT.validate_json(response.content) + + +def test_anthropic_sdk_thinking_block_reaches_hosted_vllm_as_reasoning_content(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + identity: Final = f"chatcmpl-messages-{marker}" + thinking: Final = f"The user wants Paris weather, codeword mango{marker[:4]}." + with wire_server(lambda _: Reply(body=_completion(identity))) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0) + message: Final = client.messages.create( + model=model, + max_tokens=256, + tools=_TOOLS, # pyright: ignore[reportArgumentType] # plain JSON tool definitions + messages=_tool_loop(thinking, marker), # pyright: ignore[reportArgumentType] # plain JSON content blocks + ) + assert message.id == identity + assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", "It is raining.")] + body: Final = _only_body(wire) + assert _sent_messages(body) == _expected_upstream(thinking, marker) + assert "thinking_blocks" not in json.dumps(body) and "opaque-signature" not in json.dumps(body), body + assert _spend_status(identity) == "success" + + +async def test_async_anthropic_sdk_stream_forwards_thinking_to_hosted_vllm(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + identity: Final = f"chatcmpl-messages-stream-{marker}" + thinking: Final = f"Streaming thought {marker}." + with wire_server(lambda _: _streamed_completion(identity)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + client: Final = anthropic.AsyncAnthropic( + base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0 + ) + stream: Final = await client.messages.create( + model=model, + max_tokens=256, + tools=_TOOLS, # pyright: ignore[reportArgumentType] # plain JSON tool definitions + messages=_tool_loop(thinking, marker), # pyright: ignore[reportArgumentType] # plain JSON content blocks + stream=True, + ) + events: Final = [event async for event in stream] + assert events[0].type == "message_start" and events[-1].type == "message_stop" + message_id: Final = events[0].message.id + assert "".join( + event.delta.text + for event in events + if event.type == "content_block_delta" and event.delta.type == "text_delta" + ) == ("It is raining.") + body: Final = _only_body(wire) + assert body["stream"] is True + assert _sent_messages(body) == _expected_upstream(thinking, marker) + assert _spend_status(message_id) == "success" + + +def test_redacted_thinking_alone_sends_no_reasoning_content(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with ( + wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}"))) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + _post_messages( + gateway, + model, + [ + {"role": "user", "content": f"Hello {marker}"}, + { + "role": "assistant", + "content": [ + {"type": "redacted_thinking", "data": "opaque-redacted"}, + {"type": "text", "text": "Hi."}, + ], + }, + {"role": "user", "content": "Again"}, + ], + ) + assert _sent_messages(_only_body(wire)) == [ + {"role": "user", "content": f"Hello {marker}"}, + {"role": "assistant", "content": "Hi."}, + {"role": "user", "content": "Again"}, + ] + + +def test_assistant_turn_without_thinking_sends_no_reasoning_content(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with ( + wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}"))) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + _post_messages( + gateway, + model, + [ + {"role": "user", "content": f"Hello {marker}"}, + {"role": "assistant", "content": [{"type": "text", "text": "Hi."}]}, + {"role": "user", "content": "Again"}, + ], + ) + assert _sent_messages(_only_body(wire)) == [ + {"role": "user", "content": f"Hello {marker}"}, + {"role": "assistant", "content": "Hi."}, + {"role": "user", "content": "Again"}, + ] diff --git a/tests/integration/providers/test_hosted_vllm_reasoning_content_chaos.py b/tests/integration/providers/test_hosted_vllm_reasoning_content_chaos.py new file mode 100644 index 00000000000..84a5d71e8b8 --- /dev/null +++ b/tests/integration/providers/test_hosted_vllm_reasoning_content_chaos.py @@ -0,0 +1,393 @@ +import asyncio +import json +import re +import signal +import threading +import uuid +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from queue import SimpleQueue +from types import MappingProxyType +from typing import Final, Literal +from urllib.parse import urlsplit + +import httpx +import psutil +import pytest +import yaml +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.process import owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "qwen3-reasoning-chaos" +_API_KEY: Final = "synthetic-hosted-vllm-key" +_CONFIG_MODEL: Final = "hosted-vllm-reasoning-chaos" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]]) +_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})") +_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") +_MODEL_LIST: Final = json.dumps( + {"object": "list", "data": [{"id": _BACKEND, "object": "model", "owned_by": "vllm"}]} +).encode() + +Endpoint = Literal["chat", "messages", "responses"] + + +@dataclass(frozen=True, slots=True) +class _Call: + endpoint: Endpoint + stream: bool + marker: str + + +@dataclass(frozen=True, slots=True) +class _Served: + call: _Call + status: int + text: str + + +def _thought(marker: str) -> str: + return f"private thought for {marker}" + + +def _answer(marker: str) -> str: + return f"answer marker-{marker}" + + +def _path(endpoint: Endpoint) -> str: + match endpoint: + case "chat": + return "/v1/chat/completions" + case "messages": + return "/v1/messages" + case "responses": + return "/v1/responses" + + +def _body(model: str, call: _Call) -> dict[str, JsonValue]: + question: Final = f"Question marker-{call.marker}" + common: Final[dict[str, JsonValue]] = {"model": model, "stream": call.stream, "num_retries": 0} + match call.endpoint: + case "chat": + return { + **common, + "messages": [ + {"role": "user", "content": question}, + {"role": "assistant", "content": "Working on it.", "reasoning_content": _thought(call.marker)}, + {"role": "user", "content": "Go on."}, + ], + } + case "messages": + return { + **common, + "max_tokens": 64, + "messages": [ + {"role": "user", "content": question}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": _thought(call.marker), "signature": "sig"}, + {"type": "text", "text": "Working on it."}, + ], + }, + {"role": "user", "content": "Go on."}, + ], + } + case "responses": + return { + **common, + "input": [ + {"role": "user", "content": question}, + { + "id": f"rs_{call.marker}", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": _thought(call.marker)}], + }, + {"role": "user", "content": "Go on."}, + ], + } + + +def _chat_reply(marker: str, stream: bool, abort_after: int | None = None, pause: float = 0) -> Reply: + usage: Final = {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35} + if not stream: + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{marker}", + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": _answer(marker)}, + "finish_reason": "stop", + } + ], + "usage": usage, + } + ).encode() + ) + chunk: Final = {"id": f"chatcmpl-{marker}", "object": "chat.completion.chunk", "created": 1, "model": _BACKEND} + frames: Final = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "answer "}}]}, + {**chunk, "choices": [{"index": 0, "delta": {"content": f"marker-{marker}"}}]}, + {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "usage": usage}, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(b"data: " + json.dumps(frame).encode() + b"\n\n" for frame in frames), b"data: [DONE]\n\n"), + abort_after=abort_after, + pause_between_chunks=pause, + ) + + +def _responses_reply(marker: str, stream: bool) -> Reply: + identity: Final = f"resp_upstream_{marker}" + response: Final = { + "id": identity, + "object": "response", + "created_at": 1, + "status": "completed", + "model": _BACKEND, + "output": [ + { + "id": f"msg_{marker}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": _answer(marker), "annotations": []}], + } + ], + "usage": {"input_tokens": 30, "output_tokens": 5, "total_tokens": 35}, + } + if not stream: + return Reply(body=json.dumps(response).encode()) + events: Final = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**response, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": f"msg_{marker}", + "output_index": 0, + "content_index": 0, + "delta": _answer(marker), + }, + {"type": "response.completed", "sequence_number": 2, "response": response}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events), + ) + + +def _marker_of(request: Request) -> str: + found: Final = _MARKER.search(request.body.decode()) + assert found is not None, request.body + return found.group(1) + + +def _echo(request: Request) -> Reply: + marker: Final = _marker_of(request) + stream: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True + if request.target == "/v1/responses": + return _responses_reply(marker, stream) + return _chat_reply(marker, stream) + + +def _forwarded_reasoning(request: Request) -> tuple[str, JsonValue]: + body: Final = _JSON_OBJECT.validate_json(request.body) + if request.target == "/v1/responses": + reasoning_item: Final = _MESSAGES.validate_python(body["input"])[1] + return _marker_of(request), _MESSAGES.validate_python(reasoning_item["summary"])[0]["text"] + assert request.target == "/v1/chat/completions", request.target + return _marker_of(request), _MESSAGES.validate_python(body["messages"])[1].get("reasoning_content") + + +def _assert_no_bleed(received: tuple[Request, ...], markers: frozenset[str]) -> None: + forwarded: Final = [_forwarded_reasoning(request) for request in received] + assert sorted(marker for marker, _ in forwarded) == sorted(markers) + assert all(reasoning == _thought(marker) for marker, reasoning in forwarded), forwarded + + +def _spend_statuses(model: str, expected: int) -> list[JsonValue]: + rows: Final = eventually( + lambda: read_rows('SELECT request_id, status FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda found: len(found) >= expected, + seconds=60, + ) + assert len({row["request_id"] for row in rows}) == len(rows), rows + return [row["status"] for row in rows] + + +async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served: + async with client.stream( + "POST", + _path(call.endpoint), + json=_body(model, call), + headers={"Authorization": f"Bearer {key}", "anthropic-version": "2023-06-01"}, + ) as response: + raw: Final = await response.aread() + return _Served(call=call, status=response.status_code, text=raw.decode()) + + +async def _burst( + base_url: str, key: str, model: str, calls: tuple[_Call, ...], *, tolerate_transport_errors: bool = False +) -> tuple[_Served, ...]: + async with httpx.AsyncClient(base_url=base_url, timeout=60, trust_env=False) as client: + results: Final = await asyncio.gather( + *(_send(client, key, model, call) for call in calls), return_exceptions=tolerate_transport_errors + ) + for result in results: + assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result) + return tuple(result for result in results if isinstance(result, _Served)) + + +def _calls(count: int, endpoints: tuple[Endpoint, ...], stream: Callable[[int], bool]) -> tuple[_Call, ...]: + return tuple( + _Call(endpoint=endpoints[index % len(endpoints)], stream=stream(index), marker=uuid.uuid4().hex) + for index in range(count) + ) + + +def _assert_answered_with_its_own_marker(served: _Served) -> None: + assert served.status == 200, served.text + assert set(_MARKER.findall(served.text)) == {served.call.marker}, served.text + + +async def test_concurrent_replays_across_endpoints_keep_each_reasoning_with_its_request(gateway: Gateway) -> None: + calls: Final = _calls(30, ("chat", "messages", "responses"), lambda index: index % 2 == 0) + with wire_server(_echo) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls) + assert len(served) == 30 + for item in served: + _assert_answered_with_its_own_marker(item) + _assert_no_bleed(wire.drain(), frozenset(call.marker for call in calls)) + assert _spend_statuses(model, 30) == ["success"] * 30 + + +async def test_upstream_stream_aborts_reach_callers_and_later_replays_still_forward_reasoning( + gateway: Gateway, +) -> None: + calls: Final = _calls(12, ("chat",), lambda _: True) + aborted: Final = frozenset(call.marker for index, call in enumerate(calls) if index % 3 == 0) + + def respond(request: Request) -> Reply: + marker: Final = _marker_of(request) + return _chat_reply(marker, stream=True, abort_after=0 if marker in aborted else None) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls) + assert len(served) == 12 + for item in served: + if item.call.marker in aborted: + assert item.status == 500, item.text + assert "APIConnectionError" in item.text and "marker-" not in item.text, item.text + else: + _assert_answered_with_its_own_marker(item) + assert item.text.rstrip().endswith("data: [DONE]"), item.text + recovery: Final = _Call(endpoint="chat", stream=True, marker=uuid.uuid4().hex) + (recovered,) = await _burst(str(gateway.client.base_url), gateway.key, model, (recovery,)) + _assert_answered_with_its_own_marker(recovered) + _assert_no_bleed(wire.drain(), frozenset(call.marker for call in (*calls, recovery))) + + +async def test_slow_upstream_streams_are_forwarded_once_with_their_own_reasoning(gateway: Gateway) -> None: + calls: Final = _calls(10, ("chat",), lambda _: True) + with ( + wire_server(lambda request: _chat_reply(_marker_of(request), stream=True, pause=0.3)) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls) + assert len(served) == 10 + for item in served: + _assert_answered_with_its_own_marker(item) + assert item.text.rstrip().endswith("data: [DONE]"), item.text + _assert_no_bleed(wire.drain(), frozenset(call.marker for call in calls)) + assert _spend_statuses(model, 10) == ["success"] * 10 + + +def _chaos_config(wire: Wire, tmp_path: Path) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["model_list"] = [ + { + "model_name": _CONFIG_MODEL, + "litellm_params": {"model": f"hosted_vllm/{_BACKEND}", "api_base": wire.url + "/v1", "api_key": _API_KEY}, + } + ] + path: Final = tmp_path / "hosted-vllm-reasoning-chaos.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _open_upstream_connections(pid: int, upstream: str) -> int: + port: Final = urlsplit(upstream).port + return sum( + 1 + for connection in psutil.Process(pid).net_connections(kind="tcp") + if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port + ) + + +@pytest.mark.timeout(180) +async def test_worker_sigkill_mid_burst_leaves_the_sibling_forwarding_reasoning( + gateway: Gateway, tmp_path: Path +) -> None: + calls: Final = _calls(20, ("chat",), lambda _: False) + release: Final = threading.Event() + held_markers: Final[SimpleQueue[str]] = SimpleQueue() + + def held(request: Request) -> Reply: + if (request.method, request.target) == ("GET", "/v1/models"): + return Reply(body=_MODEL_LIST) + held_markers.put(_marker_of(request)) + assert release.wait(timeout=60), "The burst was never released" + return _echo(request) + + with wire_server(held) as wire: + path: Final = _chaos_config(wire, tmp_path) + with owned_proxy_process(gateway, tmp_path, {}, config=path, workers=2) as owned: + candidate: Final = owned.gateway + workers: Final = eventually( + lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())), + lambda pids: len(pids) == 2, + seconds=30, + ) + burst: Final = asyncio.create_task( + _burst( + str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, calls, tolerate_transport_errors=True + ) + ) + await asyncio.to_thread(eventually, held_markers.qsize, lambda size: size == 20, 60) + held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire.url) for pid in workers}) + assert sum(held_by.values()) == 20, held_by + victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__) + victim: Final = psutil.Process(victim_pid) + victim.suspend() + victim.send_signal(signal.SIGKILL) + release.set() + served: Final = await burst + assert held_by[survivor_pid] >= 10, held_by + assert len(served) == held_by[survivor_pid], (held_by, len(served)) + for item in served: + _assert_answered_with_its_own_marker(item) + follow_up: Final = _Call(endpoint="chat", stream=False, marker=uuid.uuid4().hex) + (answered,) = await _burst(str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, (follow_up,)) + _assert_answered_with_its_own_marker(answered) + received: Final = wire.drain() + chats: Final = tuple(request for request in received if request.method == "POST") + probes: Final = [(request.method, request.target) for request in received if request.method != "POST"] + assert set(probes) <= {("GET", "/v1/models")}, probes + _assert_no_bleed(chats, frozenset(call.marker for call in (*calls, follow_up))) diff --git a/tests/integration/providers/test_hosted_vllm_reasoning_content_wire.py b/tests/integration/providers/test_hosted_vllm_reasoning_content_wire.py new file mode 100644 index 00000000000..858ec1af242 --- /dev/null +++ b/tests/integration/providers/test_hosted_vllm_reasoning_content_wire.py @@ -0,0 +1,565 @@ +import json +import uuid +from collections.abc import Sequence +from typing import Final + +import openai +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "qwen3-reasoning" +_FALLBACK_BACKEND: Final = "qwen3-reasoning-fallback" +_API_KEY: Final = "synthetic-hosted-vllm-key" +_REASONING: Final = "I compared the two invoices and the totals differ by 42." +_ANSWER_REASONING: Final = "The user wants the difference, which is 42." +_TOOL_CALL_ID: Final = "call_reasoning_wire_1" +_NO_CACHE: Final[dict[str, JsonValue]] = {"no-cache": True} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +def _completion(identity: str, content: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content, "reasoning_content": _ANSWER_REASONING}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35}, + } + ).encode() + + +def _streamed_completion(identity: str, content: str) -> Reply: + chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": _BACKEND} + frames: Final = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "reasoning_content": _ANSWER_REASONING}}]}, + {**chunk, "choices": [{"index": 0, "delta": {"content": content}}]}, + { + **chunk, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35}, + }, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(b"data: " + json.dumps(frame).encode() + b"\n\n" for frame in frames), b"data: [DONE]\n\n"), + ) + + +def _replayed_conversation(reasoning: JsonValue, marker: str) -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": f"Compare these invoices {marker}."}, + {"role": "assistant", "content": "Checking the totals.", "reasoning_content": reasoning}, + {"role": "user", "content": "What is the difference?"}, + ] + + +def _sent_messages(request: Request) -> list[dict[str, JsonValue]]: + return _MESSAGES.validate_python(_JSON_OBJECT.validate_json(request.body)["messages"]) + + +def _only_request(wire: Wire) -> Request: + received: Final = wire.drain() + assert [(request.method, request.target) for request in received] == [("POST", "/v1/chat/completions")] + return received[0] + + +def _spend_row(identity: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT model_group, status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (identity,), + ), + lambda found: len(found) == 1, + seconds=70, + ) + return rows[0] + + +def _model_spend_statuses(model: str) -> list[JsonValue]: + rows: Final = eventually( + lambda: read_rows('SELECT status FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda found: len(found) >= 1, + seconds=70, + ) + return [row["status"] for row in rows] + + +def _openai_client(gateway: Gateway) -> openai.OpenAI: + return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0) + + +def _async_openai_client(gateway: Gateway) -> openai.AsyncOpenAI: + return openai.AsyncOpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0) + + +def _post_chat(gateway: Gateway, model: str, messages: Sequence[dict[str, JsonValue]]) -> dict[str, JsonValue]: + response: Final = gateway.request( + "POST", "/v1/chat/completions", {"model": model, "messages": list(messages), "cache": _NO_CACHE} + ) + assert response.status_code == 200, response.text + return _JSON_OBJECT.validate_json(response.content) + + +def test_hosted_vllm_assistant_reasoning_content_reaches_the_wire(gateway: Gateway) -> None: + identity: Final = f"hosted-vllm-reasoning-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["model"] == _BACKEND + assert body["messages"] == [ + {"role": "user", "content": "Compare these invoices."}, + { + "role": "assistant", + "content": "Checking the totals.", + "reasoning_content": _REASONING, + "tool_calls": [ + { + "id": _TOOL_CALL_ID, + "type": "function", + "function": {"name": "lookup_invoice", "arguments": json.dumps({"id": "inv-7"})}, + } + ], + }, + {"role": "tool", "tool_call_id": _TOOL_CALL_ID, "content": "invoice total is 1042"}, + ], body["messages"] + return Reply(body=_completion(identity, "The totals differ by 42.")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [ + {"role": "user", "content": "Compare these invoices."}, + { + "role": "assistant", + "content": "Checking the totals.", + "reasoning_content": _REASONING, + "tool_calls": [ + { + "id": _TOOL_CALL_ID, + "type": "function", + "function": {"name": "lookup_invoice", "arguments": json.dumps({"id": "inv-7"})}, + } + ], + }, + {"role": "tool", "tool_call_id": _TOOL_CALL_ID, "content": "invoice total is 1042"}, + ], + }, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["id"] == identity + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/chat/completions")] + + +def test_openai_sdk_replayed_reasoning_reaches_hosted_vllm_and_is_billed_once(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + identity: Final = f"chatcmpl-sdk-{marker}" + with wire_server(lambda _: Reply(body=_completion(identity, "They differ by 42."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + completion: Final = _openai_client(gateway).chat.completions.create( + model=model, + messages=_replayed_conversation(_REASONING, marker), # pyright: ignore[reportArgumentType] # reasoning_content is a provider extension the SDK types omit + ) + assert completion.id == identity + assert completion.choices[0].message.content == "They differ by 42." + assert (completion.choices[0].message.model_extra or {})["reasoning_content"] == _ANSWER_REASONING + assert _sent_messages(_only_request(wire)) == _replayed_conversation(_REASONING, marker) + assert _spend_row(identity) == { + "model_group": model, + "status": "success", + "prompt_tokens": 30, + "completion_tokens": 5, + } + + +async def test_async_openai_sdk_stream_forwards_replayed_reasoning_to_hosted_vllm(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + identity: Final = f"chatcmpl-stream-{marker}" + with wire_server(lambda _: _streamed_completion(identity, "They differ by 42.")) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + stream: Final = await _async_openai_client(gateway).chat.completions.create( + model=model, + messages=_replayed_conversation(_REASONING, marker), # pyright: ignore[reportArgumentType] # reasoning_content is a provider extension the SDK types omit + stream=True, + stream_options={"include_usage": True}, + ) + chunks: Final = [chunk async for chunk in stream] + assert {chunk.id for chunk in chunks} == {identity} + assert "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) == ( + "They differ by 42." + ) + sent: Final = _only_request(wire) + assert _JSON_OBJECT.validate_json(sent.body)["stream"] is True + assert _sent_messages(sent) == _replayed_conversation(_REASONING, marker) + assert _spend_row(identity)["status"] == "success" + + +def test_each_replayed_turn_keeps_its_own_reasoning_in_order(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + conversation: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": f"Plan the migration {marker}."}, + {"role": "assistant", "content": "Step one.", "reasoning_content": f"first thought {marker}"}, + {"role": "user", "content": "Continue."}, + {"role": "assistant", "content": "Step two.", "reasoning_content": f"second thought {marker}"}, + {"role": "user", "content": "Summarize."}, + ] + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + assert _post_chat(gateway, model, conversation)["id"] == f"chatcmpl-{marker}" + assert _sent_messages(_only_request(wire)) == conversation + + +@pytest.mark.parametrize( + ("reasoning", "forwarded"), + [ + pytest.param("", "", id="empty-string-forwarded"), + pytest.param("x" * 5120, "x" * 5120, id="5kb-string-forwarded-intact"), + pytest.param(None, None, id="null-dropped"), + pytest.param(42, None, id="int-dropped"), + pytest.param(["step one", "step two"], None, id="list-dropped"), + pytest.param({"text": "step one"}, None, id="object-dropped"), + ], +) +def test_only_string_reasoning_content_is_forwarded_to_hosted_vllm( + gateway: Gateway, reasoning: JsonValue, forwarded: str | None +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + assert _post_chat(gateway, model, _replayed_conversation(reasoning, marker))["id"] == f"chatcmpl-{marker}" + sent_assistant: Final = _sent_messages(_only_request(wire))[1] + expected_assistant: Final[dict[str, JsonValue]] = {"role": "assistant", "content": "Checking the totals."} + assert sent_assistant == ( + expected_assistant if forwarded is None else {**expected_assistant, "reasoning_content": forwarded} + ) + + +def test_assistant_turn_without_reasoning_gets_no_reasoning_key(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + conversation: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": f"Hello {marker}"}, + {"role": "assistant", "content": "Hi there."}, + {"role": "user", "content": "Again"}, + ] + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Hello again."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + _post_chat(gateway, model, conversation) + assert _sent_messages(_only_request(wire)) == conversation + + +def test_same_reasoning_on_two_turns_is_forwarded_on_both(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + reasoning: Final = f"repeated thought {marker}" + conversation: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": "One"}, + {"role": "assistant", "content": "First.", "reasoning_content": reasoning}, + {"role": "user", "content": "Two"}, + {"role": "assistant", "content": "Second.", "reasoning_content": reasoning}, + {"role": "user", "content": "Three"}, + ] + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Third."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + _post_chat(gateway, model, conversation) + assert _sent_messages(_only_request(wire)) == conversation + + +def test_thinking_blocks_are_stripped_while_reasoning_content_is_kept(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + _post_chat( + gateway, + model, + [ + {"role": "user", "content": f"Hello {marker}"}, + { + "role": "assistant", + "content": "Hi.", + "reasoning_content": _REASONING, + "thinking_blocks": [{"type": "thinking", "thinking": _REASONING, "signature": "sig"}], + }, + {"role": "user", "content": "Again"}, + ], + ) + assert _sent_messages(_only_request(wire))[1] == { + "role": "assistant", + "content": "Hi.", + "reasoning_content": _REASONING, + } + + +def test_list_content_is_flattened_while_reasoning_content_is_kept(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + _post_chat( + gateway, + model, + [ + {"role": "user", "content": f"Hello {marker}"}, + { + "role": "assistant", + "content": [{"type": "text", "text": "Part one."}, {"type": "text", "text": "Part two."}], + "reasoning_content": _REASONING, + }, + {"role": "user", "content": "Again"}, + ], + ) + assert _sent_messages(_only_request(wire))[1] == { + "role": "assistant", + "content": "Part one.\nPart two.", + "reasoning_content": _REASONING, + } + + +def test_unauthenticated_replay_is_rejected_before_hosted_vllm(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(lambda _: Reply(body=_completion(f"chatcmpl-{marker}", "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": _replayed_conversation(_REASONING, marker)}, + key=f"sk-not-a-key-{marker}", + ) + assert response.status_code == 401, response.text + assert wire.drain() == () + + +def test_hosted_vllm_auth_error_reaches_the_caller_after_one_attempt_with_reasoning(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + error_message: Final = f"invalid api key for deployment {marker}" + reply: Final = Reply( + status=401, + body=json.dumps({"error": {"message": error_message, "type": "authentication_error"}}).encode(), + ) + with wire_server(lambda _: reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": _replayed_conversation(_REASONING, marker), "cache": _NO_CACHE}, + ) + assert response.status_code == 401, response.text + assert error_message in response.text, response.text + assert _sent_messages(_only_request(wire)) == _replayed_conversation(_REASONING, marker) + + +def test_fallback_attempt_replays_reasoning_to_the_second_deployment(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + + def respond(request: Request) -> Reply: + if _JSON_OBJECT.validate_json(request.body)["model"] == _BACKEND: + return Reply(status=500, body=b'{"error": {"message": "primary deployment is down"}}') + return Reply(body=_completion(f"chatcmpl-fallback-{marker}", "Recovered.")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + primary: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + fallback: Final = scenario.model( + model=f"hosted_vllm/{_FALLBACK_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": primary, + "messages": _replayed_conversation(_REASONING, marker), + "fallbacks": [fallback], + "num_retries": 0, + "cache": _NO_CACHE, + }, + ) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content)["id"] == f"chatcmpl-fallback-{marker}" + attempts: Final = wire.drain() + assert [_JSON_OBJECT.validate_json(attempt.body)["model"] for attempt in attempts] == [ + _BACKEND, + _FALLBACK_BACKEND, + ] + assert [_sent_messages(attempt) for attempt in attempts] == [ + _replayed_conversation(_REASONING, marker), + _replayed_conversation(_REASONING, marker), + ] + + +def test_identical_uncached_replays_are_each_forwarded_and_billed_once(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + identities: Final = iter((f"chatcmpl-first-{marker}", f"chatcmpl-second-{marker}")) + with wire_server(lambda _: Reply(body=_completion(next(identities), "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + first: Final = _post_chat(gateway, model, _replayed_conversation(_REASONING, marker)) + second: Final = _post_chat(gateway, model, _replayed_conversation(_REASONING, marker)) + assert (first["id"], second["id"]) == (f"chatcmpl-first-{marker}", f"chatcmpl-second-{marker}") + assert [_sent_messages(request) for request in wire.drain()] == [ + _replayed_conversation(_REASONING, marker), + _replayed_conversation(_REASONING, marker), + ] + assert _spend_row(f"chatcmpl-first-{marker}")["status"] == "success" + assert _spend_row(f"chatcmpl-second-{marker}")["status"] == "success" + + +def test_cached_replay_hits_only_for_the_same_reasoning(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + identities: Final = iter((f"chatcmpl-cached-{marker}", f"chatcmpl-other-{marker}")) + with wire_server(lambda _: Reply(body=_completion(next(identities), "Done."))) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + + def ask(reasoning: str) -> dict[str, JsonValue]: + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": _replayed_conversation(reasoning, marker)}, + ) + assert response.status_code == 200, response.text + return _JSON_OBJECT.validate_json(response.content) + + assert ask(_REASONING)["id"] == f"chatcmpl-cached-{marker}" + assert ask(_REASONING)["id"] == f"chatcmpl-cached-{marker}" + assert ask(f"a different thought {marker}")["id"] == f"chatcmpl-other-{marker}" + assert [_sent_messages(request)[1].get("reasoning_content") for request in wire.drain()] == [ + _REASONING, + f"a different thought {marker}", + ] + + +def _responses_input(marker: str) -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": f"Compare these invoices {marker}."}, + { + "id": f"rs_{marker}", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": _REASONING}], + }, + { + "id": f"msg_prior_{marker}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Checking the totals.", "annotations": []}], + }, + {"role": "user", "content": "What is the difference?"}, + ] + + +def _responses_reply(identity: str, stream: bool) -> Reply: + response: Final = { + "id": identity, + "object": "response", + "created_at": 1, + "status": "completed", + "model": _BACKEND, + "output": [ + { + "id": "msg_" + identity, + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "They differ by 42.", "annotations": []}], + } + ], + "usage": {"input_tokens": 30, "output_tokens": 5, "total_tokens": 35}, + } + if not stream: + return Reply(body=json.dumps(response).encode()) + events: Final = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**response, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_" + identity, + "output_index": 0, + "content_index": 0, + "delta": "They differ by 42.", + }, + {"type": "response.completed", "sequence_number": 2, "response": response}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events), + ) + + +def _only_responses_body(wire: Wire) -> dict[str, JsonValue]: + received: Final = wire.drain() + assert [(request.method, request.target) for request in received] == [("POST", "/v1/responses")] + return _JSON_OBJECT.validate_json(received[0].body) + + +def test_openai_sdk_responses_replay_reaches_hosted_vllm_with_its_reasoning_item(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(lambda _: _responses_reply(f"resp_upstream_{marker}", stream=False)) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + response: Final = _openai_client(gateway).responses.create( + model=model, + input=_responses_input(marker), # pyright: ignore[reportArgumentType] # plain JSON input items + ) + assert response.output_text == "They differ by 42." + assert _only_responses_body(wire)["input"] == _responses_input(marker) + assert _spend_row(response.id) == { + "model_group": model, + "status": "success", + "prompt_tokens": 30, + "completion_tokens": 5, + } + + +async def test_async_openai_sdk_responses_stream_reaches_hosted_vllm_with_its_reasoning_item( + gateway: Gateway, +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(lambda _: _responses_reply(f"resp_upstream_{marker}", stream=True)) as wire: + with gateway.scenario() as scenario: + model: Final = scenario.model(model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY) + stream: Final = await _async_openai_client(gateway).responses.create( + model=model, + input=_responses_input(marker), # pyright: ignore[reportArgumentType] # plain JSON input items + stream=True, + ) + events: Final = [event async for event in stream] + assert [event.type for event in events] == [ + "response.created", + "response.output_text.delta", + "response.completed", + ] + completed: Final = events[-1] + assert completed.type == "response.completed" + body: Final = _only_responses_body(wire) + assert body["stream"] is True + assert body["input"] == _responses_input(marker) + assert completed.response.output_text == "They differ by 42." + assert _model_spend_statuses(model) == ["success"] diff --git a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index 1cc6a1457fc..c792d5dffcd 100644 --- a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -1,6 +1,8 @@ import json from unittest.mock import MagicMock, patch +import pytest + from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, @@ -112,10 +114,10 @@ def test_hosted_vllm_supports_thinking(): assert optional_params["reasoning_effort"] == "low" -def test_hosted_vllm_thinking_blocks_prepended_to_assistant_content(): +def test_hosted_vllm_reasoning_content_kept_and_thinking_blocks_removed(): """ - Test that thinking_blocks on assistant messages are removed and content - stays a string for vLLM compatibility. + Test that reasoning_content on assistant messages is forwarded to vLLM + while thinking_blocks are removed and content stays a string. """ config = HostedVLLMChatConfig() messages = [ @@ -152,7 +154,36 @@ def test_hosted_vllm_thinking_blocks_prepended_to_assistant_content(): assert isinstance(assistant_msg["content"], str) assert assistant_msg["content"] == "Here is my answer." assert "thinking_blocks" not in assistant_msg - assert "reasoning_content" not in assistant_msg + assert assistant_msg["reasoning_content"] == "Let me reason about this..." + + +@pytest.mark.parametrize( + ("reasoning_content", "expected"), + [ + ("step one, then step two", "step one, then step two"), + ("", ""), + (None, "absent"), + (42, "absent"), + (["step one", "step two"], "absent"), + ({"text": "step one"}, "absent"), + ], +) +def test_hosted_vllm_forwards_only_string_reasoning_content(reasoning_content, expected): + config = HostedVLLMChatConfig() + transformed = config.transform_request( + model="hosted_vllm/qwen3", + messages=[ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi", "reasoning_content": reasoning_content}, + {"role": "user", "content": "Again"}, + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + assistant_msg = transformed["messages"][1] + assert assistant_msg.get("reasoning_content", "absent") == expected + assert assistant_msg["content"] == "Hi" def test_hosted_vllm_thinking_blocks_with_list_content(): From f285229b511390f124aa92706101fe6437863dda Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:49:18 -0700 Subject: [PATCH 4/7] fix(proxy): delete large teams without per-member transaction fan-out (#42998) * fix(proxy): delete large teams without per-member transaction fan-out Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): evict email-only member caches and reset team members metric on delete Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep new delete-team literals within the LIT002 ceiling Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): resolve deleted-team member ids before the locked delete Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): resolve email-only deleted-team members with one case-insensitive lookup `_deleted_team_member_user_ids` looked each email-only roster entry up with its own `find_users_by_email` call inside an unbounded `asyncio.gather`: one exact-match query per email, so a large roster fanned out against the pool again and a roster email that differed in case from its user row was missed. Add `UserRepository.find_by_emails`, a single case-insensitive `in` query, and call it once before the locked delete. `management_helpers/utils.py` goes back to its main-branch shape since the single-email helper no longer needs exporting. * fix(repositories): slice find_by_emails into bounded IN statements The unbounded-IN lint flagged the case-insensitive email lookup added for /team/delete cache eviction. chunked_in.find_many_in cannot carry Prisma's insensitive mode, so the repository slices the deduplicated list into IN_LIST_CHUNK_SIZE statements itself and concatenates the pages. Empty input still returns () without a query. * fix(proxy): delete a team once when /team/delete repeats its id The audit sent {"team_ids": [T, T]}: main answered 400 "User not found in team" after deleting the keys and memberships and writing two tombstones, leaving the team row behind; this branch answered 200 but still wrote the tombstone, audit row and eviction twice. DeleteTeamRequest now collapses repeated ids in order, so every later step sees each team once and the response lists each deleted team once. * test(integration): audit cells for /team/delete on large, legacy and concurrent teams Thirty-eight deterministic cells in tests/integration/management/ (the CircleCI integration-management group) covering the /team/delete happy, sad, edge and chaos rows: 250 members against a pool limit of five on two workers, the advisory-lock wait, email-only legacy roster entries in every casing, member and team cache eviction on both proxies for every client and endpoint, the Prometheus gauge, audit rows, malformed and duplicate input, the route gate, and a worker kill, a Redis outage and a proxy restart mid-burst. Every cell runs against the real proxy, Postgres and Redis with the scripted upstream; no component is mocked. On the merge base the rows this fix changes are red (P2028 on the 250-member team, two lock waiters, case-mismatched email lookups, duplicate ids, orphaned LiteLLM_UserTable.teams references under a concurrent burst); on the tip every cell is green twice with identical selections. Two pre-existing behaviours are pinned as observed rather than fixed here: a roster entry with neither user_id nor user_email answers 500, and the LiteLLM_DeletedTeamTable row is committed before the locked transaction, so a delete that dies in between leaves a tombstone for a live team and the retry adds a second. * test(integration): pin each chaos outage to a live /team/delete The three chaos cells applied the outage once three deletes had answered, which on a fast run let the whole burst finish before the worker kill, Redis stop or SIGTERM landed, so the cells passed without exercising the failure. Each cell now holds the first team's advisory lock from a test-owned transaction, waits until that team's delete is queued behind it in Postgres with its request unanswered, applies the outage, and only then releases the lock, so an in-flight delete meets the failure on every run and both legs. The pinned team's outcome and the number of deletes answered before the outage are recorded as junit properties (pinned_delete, answered_before_outage). --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: ryan-crabbe-berri --- litellm/proxy/_types.py | 8 +- .../management_endpoints/team_endpoints.py | 88 ++- litellm/repositories/user_repository.py | 29 +- tests/e2e/coverage_registry/mgmt.yaml | 1 + tests/e2e/management/management_client.py | 21 + tests/e2e/management/test_management_e2e.py | 39 ++ tests/e2e/models.py | 2 +- .../management/test_team_delete_chaos.py | 499 ++++++++++++++ .../management/test_team_delete_inputs.py | 330 +++++++++ .../test_team_delete_large_membership.py | 634 ++++++++++++++++++ .../test_team_delete_member_cache_eviction.py | 446 ++++++++++++ .../management/test_team_delete_prometheus.py | 122 ++++ .../test_team_endpoints.py | 117 +++- tests/unit/repositories/test_repositories.py | 27 + 14 files changed, 2331 insertions(+), 32 deletions(-) create mode 100644 tests/integration/management/test_team_delete_chaos.py create mode 100644 tests/integration/management/test_team_delete_inputs.py create mode 100644 tests/integration/management/test_team_delete_large_membership.py create mode 100644 tests/integration/management/test_team_delete_member_cache_eviction.py create mode 100644 tests/integration/management/test_team_delete_prometheus.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 6c9e1511b00..9c7dece6894 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1,7 +1,7 @@ import enum import json import os -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from datetime import datetime from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NamedTuple, TypeAlias @@ -2241,6 +2241,12 @@ class ResetTeamBudgetRequest(LiteLLMPydanticObjectBase): class DeleteTeamRequest(LiteLLMPydanticObjectBase): team_ids: list[str] # required + @field_validator("team_ids") + @classmethod + def distinct_team_ids(cls, team_ids: Sequence[str]) -> list[str]: + """One delete per team: a repeated id would otherwise write its tombstone and audit row twice.""" + return list(dict.fromkeys(team_ids)) + class BlockTeamRequest(LiteLLMPydanticObjectBase): team_id: str # required diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 6cec3e714ec..f0e59389d48 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -4517,27 +4517,13 @@ async def delete_team( llm_router=llm_router, ) - # ## DELETE TEAM MEMBERSHIPS - for team_row in team_rows: - ### get all team members - team_members = team_row.members_with_roles - ### call team_member_delete for each team member - tasks = [] - for team_member in team_members: - tasks.append( - _team_member_delete( - data=TeamMemberDeleteRequest( - team_id=team_row.team_id, - user_id=team_member.user_id, - user_email=team_member.user_email, - ), - user_api_key_dict=user_api_key_dict, - ) - ) - await asyncio.gather(*tasks) - await _sweep_deleted_team_references(team_ids=data.team_ids, prisma_client=prisma_client) + member_ids_per_team: Final = await _resolve_deleted_team_member_user_ids( + teams=team_rows, + prisma_client=prisma_client, + ) + ## DELETE TEAMS # Both the delete and the reconcile sweep run under every team's advisory lock # (TEAM_ADVISORY_LOCK_SQL, the same one /team/member_add takes before its own writes), @@ -4565,8 +4551,15 @@ async def delete_team( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + await _invalidate_deleted_team_member_cache( + member_ids_per_team=member_ids_per_team, + user_api_key_cache=user_api_key_cache, + ) for deleted_team in team_rows: + _emit_team_members_metric( + deleted_team.model_copy(update={"members_with_roles": ()}) # mutable-ok: pydantic update payload + ) await sync_team_access_group_membership(prisma_client=prisma_client, team_id=deleted_team.team_id) return deleted_teams @@ -4641,6 +4634,63 @@ async def _invalidate_deleted_team_cache( ) +async def _invalidate_deleted_team_member_cache( + member_ids_per_team: Sequence[tuple[str, Sequence[str]]], + user_api_key_cache: UserApiKeyCache, +) -> None: + for team_id, member_user_ids in member_ids_per_team: + await _evict_deleted_team_member_cache( + team_id=team_id, + member_user_ids=member_user_ids, + user_api_key_cache=user_api_key_cache, + ) + + +async def _evict_deleted_team_member_cache( + team_id: str, + member_user_ids: Sequence[str], + user_api_key_cache: UserApiKeyCache, +) -> None: + await evict_and_broadcast(cache_keys=tuple(member_user_ids), user_api_key_cache=user_api_key_cache) + await asyncio.gather( + *( + invalidate_team_member_spend_state( + user_id=user_id, + team_id=team_id, + user_api_key_cache=user_api_key_cache, + ) + for user_id in member_user_ids + ) + ) + + +async def _resolve_deleted_team_member_user_ids( + teams: Sequence[LiteLLM_TeamTable], + prisma_client: PrismaClient, +) -> tuple[tuple[str, tuple[str, ...]], ...]: + resolved: Final = await asyncio.gather( + *(_deleted_team_member_user_ids(team=team, prisma_client=prisma_client) for team in teams) + ) + return tuple(zip((team.team_id for team in teams), resolved)) + + +async def _deleted_team_member_user_ids(team: LiteLLM_TeamTable, prisma_client: PrismaClient) -> tuple[str, ...]: + roster_user_ids: Final = frozenset( + member.user_id for member in team.members_with_roles if member.user_id is not None + ) + email_only_member_emails: Final = frozenset( + member.user_email + for member in team.members_with_roles + if member.user_id is None and member.user_email is not None + ) + if not email_only_member_emails: + return tuple(sorted(roster_user_ids)) + # One case-insensitive lookup for the whole roster. A per-email fan-out would size the + # query count by team membership, the same shape as the P2028 fan-out this path removed. + email_only_users: Final = await UserRepository(prisma_client).find_by_emails(sorted(email_only_member_emails)) + return tuple(sorted(roster_user_ids.union(user.user_id for user in email_only_users))) + + def _transform_teams_to_deleted_records( teams: list[LiteLLM_TeamTable], user_api_key_dict: UserAPIKeyAuth, diff --git a/litellm/repositories/user_repository.py b/litellm/repositories/user_repository.py index 4a2aea46197..7bf516a2bc1 100644 --- a/litellm/repositories/user_repository.py +++ b/litellm/repositories/user_repository.py @@ -3,13 +3,15 @@ User repository for database operations on LiteLLM_UserTable. """ import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence +from itertools import chain from typing import TYPE_CHECKING, Final from pydantic import TypeAdapter from litellm.models.user import LiteLLM_UserTable, SCIMPlaceholder from litellm.repositories.base_repository import BaseRepository, DbRecord, record_to_dict +from litellm.repositories.chunked_in import IN_LIST_CHUNK_SIZE from litellm.repositories.prisma_protocols import TableActions if TYPE_CHECKING: @@ -71,6 +73,31 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): records: Final = await self.find_many(where={"user_email": user_email}) return records[0] if records else None + async def find_by_emails(self, user_emails: Sequence[str]) -> Sequence[LiteLLM_UserTable]: + """Every user whose email matches one of ``user_emails``, ignoring case. + + A roster entry stored by email can differ in case from its user row (member_add + resolves emails case-insensitively), so an exact match would miss it. The list goes + out in slices of ``IN_LIST_CHUNK_SIZE`` so one statement stays under Postgres's + bind-parameter cap; ``chunked_in.find_many_in`` cannot carry the insensitive mode. + """ + unique: Final = sorted(frozenset(user_emails)) + pages: Final = tuple( + [ + await self.find_many( + where={ # mutable-ok: Prisma query filters are dict-shaped + "user_email": { # mutable-ok: Prisma query filters are dict-shaped + # bounded-ok: sliced to IN_LIST_CHUNK_SIZE values per statement + "in": unique[start : start + IN_LIST_CHUNK_SIZE], + "mode": "insensitive", + } + } + ) + for start in range(0, len(unique), IN_LIST_CHUNK_SIZE) + ] + ) + return tuple(chain.from_iterable(pages)) + async def find_by_sso_id(self, sso_user_id: str) -> LiteLLM_UserTable | None: """Find a user by SSO ID.""" return await self.find_by_id(sso_user_id, id_field="sso_user_id") diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index e1a840b1239..bc728efacd4 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -35,6 +35,7 @@ - {id: mgmt.team.update.team_admin_cannot_grow_budget, module: mgmt, tier: P0, surface: api, assertions: [team_admin_cannot_grow_budget], source: "team_endpoints.py:1203", fail_before_fix: proven, rationale: "With max_budget enabled, a team admin may keep or lower its team's budget; raising or removing it is 403 and writes nothing, also under an organization's larger cap"} - {id: mgmt.team.update.team_admin_resend_keeps_budget_reset, module: mgmt, tier: P1, surface: api, assertions: [team_admin_resend_keeps_budget_reset], source: "team_admin_field_permissions.py:147", fail_before_fix: proven, rationale: "A team admin resending unchanged budget settings with an enabled field must not push the team's budget reset times back"} - {id: mgmt.team.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "team_endpoints.py:1750", rationale: "Deletion prevents key access"} +- {id: mgmt.team.delete.membership_larger_than_db_pool, module: mgmt, tier: P0, surface: api, assertions: [membership_larger_than_db_pool], source: "team_endpoints.py:4362", rationale: "Deleting a team with more members than the Prisma connection pool still completes instead of exhausting the pool and answering 500", fail_before_fix: proven} - {id: mgmt.team.block.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "team_endpoints.py", rationale: "Block suspends all members"} - {id: mgmt.team.info.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "team_endpoints.py:2244", rationale: "Metadata+members+budgets"} - {id: mgmt.team.daily_activity.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "vendor testing strategy §9.20 / LIT-4778", rationale: "GET /team/daily/activity returns results+metadata for a valid date range"} diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index 7366695c0d1..3da9bea12a3 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -411,6 +411,27 @@ class ManagementClient: assert last is not None raise AssertionError(last) + def add_team_members(self, team_id: str, members: list[TeamMemberEntry]) -> None: + """Bulk form of /team/member_add: `member` accepts a list, so one call + seeds a whole roster the way an admin import does.""" + _ = unwrap( + self.proxy.transport.post( + "/team/member_add", + headers=self.proxy.management_headers(), + json=TeamMemberAddBody(team_id=team_id, member=members), + response_type=NoBody, + ) + ) + + def delete_team_status(self, team_id: str) -> StreamingResponse: + """POST /team/delete judged by HTTP outcome: the raw status and body, so a + test can assert on what a caller actually sees when the delete fails.""" + return self.proxy.transport.send( + "/team/delete", + headers=self.proxy.management_headers(), + json=TeamDeleteBody(team_ids=[team_id]), + ) + def delete_team_member(self, team_id: str, user_id: str) -> None: _ = unwrap( self.proxy.transport.post( diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index da0fc37aff8..908eb752611 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -37,6 +37,7 @@ from models import ( OrgUpdateBody, TagListEntry, TagNewBody, + TeamMemberEntry, TeamNewBody, TeamUpdateBody, UserNewBody, @@ -48,6 +49,7 @@ pytestmark = pytest.mark.e2e REGENERATE_GRACE_PERIOD = "15s" REGENERATE_GRACE_SECONDS = 15.0 +TEAM_DELETE_POOL_OVERFLOW_MEMBERS = 250 def _poll[T](client: ManagementClient, attempt: Callable[[], T | None], failure: str) -> T: @@ -479,6 +481,43 @@ class TestTeamRoutes: client, rejected, "team-bound key was still accepted on chat (never rejected 401) after team deletion" ) + @pytest.mark.covers("mgmt.team.delete.membership_larger_than_db_pool") + def test_team_delete_succeeds_for_team_larger_than_db_pool( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + """Customer repro: /team/delete fans one transaction per member out over a + Prisma pool of 10 connections, each queued on the team's advisory lock, + so a team bigger than the pool must still delete cleanly instead of + answering 500 P2028.""" + team_id = _create_team(client, resources, f"e2e-mgmt-team-{unique_marker()}", []) + user_ids = tuple( + _create_user( + client, + resources, + UserNewBody( + user_email=f"e2e-mgmt-bulk-{i}-{unique_marker()}@example.com", + user_role="internal_user", + ), + ) + for i in range(TEAM_DELETE_POOL_OVERFLOW_MEMBERS) + ) + client.add_team_members(team_id, [TeamMemberEntry(role="user", user_id=user_id) for user_id in user_ids]) + seated = len(client.team_info(team_id).members_with_roles) + assert seated >= len(user_ids), ( + f"/team/info lists {seated} members after the bulk /team/member_add, expected at least {len(user_ids)}" + ) + + outcome = client.delete_team_status(team_id) + + assert outcome.status_code == 200, ( + f"/team/delete on a {len(user_ids)}-member team must succeed, got " + f"{outcome.status_code}: {outcome.body[:500]}" + ) + probe = client.team_info_status(team_id) + assert probe.status_code == 404, ( + f"deleted team {team_id} still resolves: /team/info returned {probe.status_code}: {probe.body[:300]}" + ) + @pytest.mark.covers("mgmt.team.member_add.persists") def test_member_add_and_delete_persist_to_team_info( self, client: ManagementClient, resources: ResourceManager diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 0383da48c43..4f8f6926095 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1485,7 +1485,7 @@ class TeamInfoResponse(BaseModel): class TeamMemberAddBody(BaseModel): team_id: str - member: TeamMemberEntry + member: TeamMemberEntry | list[TeamMemberEntry] class TeamMemberDeleteBody(BaseModel): diff --git a/tests/integration/management/test_team_delete_chaos.py b/tests/integration/management/test_team_delete_chaos.py new file mode 100644 index 00000000000..ebe515f59ec --- /dev/null +++ b/tests/integration/management/test_team_delete_chaos.py @@ -0,0 +1,499 @@ +"""Chaos rows for ``/team/delete`` on an owned two-worker proxy: C1 worker kill, C2 Redis outage, C3 proxy restart. + +Each leg creates 24 teams through the owned proxy (two internal users per team in one bulk +``/team/member_add``, plus one team key), then deletes all 24 in a 24-thread burst and breaks the +infrastructure while a delete is provably in flight: the test holds the first team's advisory lock +from its own transaction, waits until that team's delete is queued behind it inside Postgres with +its request unanswered, applies the failure once the third of the other deletes has answered, and +only then releases the lock. The outage therefore overlaps a live delete on every run and both legs, +and the pinned delete finishes, or is dropped, under the failure: + +- C1 SIGKILLs one uvicorn worker child; the survivor still answers ``/health/readiness`` and uvicorn + respawns the worker. +- C2 shuts the owned Redis down; ``/cache/ping`` reports it, the deletes keep answering 200 because + cache eviction and the invalidation broadcast are best-effort, then Redis comes back. +- C3 SIGTERMs the owned proxy root and a fresh proxy starts on the same database. + +After recovery the burst outcomes (status or transport error per team) are recorded, every team whose +row survived is deleted once more, and the invariants must hold for every team: no ``LiteLLM_TeamTable`` +row, no ``LiteLLM_TeamMembership`` row, no ``LiteLLM_UserTable.teams`` entry naming it, its key gone +from ``LiteLLM_VerificationToken``, and one ``LiteLLM_DeletedTeamTable`` row per attempt that reached +the tombstone write. Both legs commit that tombstone before the locked transaction that removes the +team, so an attempt that died in between leaves a tombstone for a live team and the retry adds a +second; that count is pinned as observed (pre-existing, outside this PR's diff, recorded in the audit +report) and the affected teams are recorded as ``double_tombstones``. Teams found half-deleted before +the retry are recorded as ``partial_states_before_retry`` and named in any failure; the pinned team's +outcome is recorded as ``pinned_delete`` and the answers the outage interrupted as +``answered_before_outage``. + +Nothing sleeps, and only processes the test started are signalled. +""" + +from __future__ import annotations + +import os +import threading +import uuid +from collections import Counter +from collections.abc import Callable, Iterator, Sequence +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import httpx +import psutil +import psycopg +import pytest + +from tests.integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + delete_key_if_present, + eventually, + string_value, +) +from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy_process +from tests.integration._support.redis_process import owned_redis + +RecordProperty = Callable[[str, object], None] + +TEAMS: Final = 24 +MEMBERS_PER_TEAM: Final = 2 +CHAOS_AFTER_ANSWERS: Final = 3 +WORKERS: Final = 2 +DELETE_TIMEOUT_SECONDS: Final = 60 +REMOVE_FROM_ENVIRONMENT: Final = ("DATABASE_URL_READ_REPLICA",) + +TEAM_SQL: Final = 'SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s' +TOMBSTONE_SQL: Final = 'SELECT id FROM "LiteLLM_DeletedTeamTable" WHERE team_id = %s' +MEMBERSHIP_SQL: Final = 'SELECT user_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s' +REFERENCING_USERS_SQL: Final = 'SELECT user_id FROM "LiteLLM_UserTable" WHERE %s = ANY(teams)' +TOKEN_SQL: Final = 'SELECT token FROM "LiteLLM_VerificationToken" WHERE token = %s' +TAKE_TEAM_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext(%s))" +# Sessions blocked on an advisory lock the given backend holds: the pinned team's delete, on either leg. +WAITERS_ON_HELD_LOCK_SQL: Final = """ +SELECT count(*)::int AS waiting +FROM pg_locks waiter +JOIN pg_stat_activity session ON session.pid = waiter.pid +WHERE waiter.locktype = 'advisory' + AND NOT waiter.granted + AND session.wait_event_type = 'Lock' + AND session.query ILIKE %s + AND (waiter.classid, waiter.objid, waiter.objsubid) IN ( + SELECT held.classid, held.objid, held.objsubid + FROM pg_locks held + WHERE held.locktype = 'advisory' AND held.granted AND held.pid = %s::int + ) +""" + + +@dataclass(frozen=True, slots=True) +class Team: + team_id: str + members: tuple[str, ...] + hashed_key: str + + +@dataclass(frozen=True, slots=True) +class Outcome: + """One burst delete: the HTTP status, or ``None`` with the transport error's class and message.""" + + team_id: str + status: int | None + detail: str + + @property + def label(self) -> str: + return str(self.status) if self.status is not None else self.detail.split(":", 1)[0] + + @property + def answered_or_dropped(self) -> bool: + """200, a 5xx from a dying process, or a transport error; a 4xx would mean a wrong delete.""" + return self.status is None or self.status == 200 or self.status >= 500 + + +@dataclass(frozen=True, slots=True) +class TeamState: + team_id: str + row_present: bool + tombstones: int + memberships: tuple[str, ...] + referencing_users: tuple[str, ...] + key_present: bool + + @property + def clean(self) -> bool: + """Row, memberships, ``teams`` references and key all gone; tombstones are counted per attempt.""" + return not self.row_present and not self.memberships and not self.referencing_users and not self.key_present + + @property + def untouched(self) -> bool: + return self.row_present and self.tombstones == 0 and self.key_present + + @property + def partial(self) -> bool: + return not (self.clean and self.tombstones == 1) and not self.untouched + + def describe(self) -> str: + return ( + f"{self.team_id}: row={'present' if self.row_present else 'gone'} tombstones={self.tombstones} " + f"memberships={len(self.memberships)} referencing_users={len(self.referencing_users)} " + f"key={'present' if self.key_present else 'gone'}" + ) + + +def _state(team: Team) -> TeamState: + return TeamState( + team.team_id, + row_present=bool(read_rows(TEAM_SQL, (team.team_id,))), + tombstones=len(read_rows(TOMBSTONE_SQL, (team.team_id,))), + memberships=tuple(string_value(row["user_id"]) for row in read_rows(MEMBERSHIP_SQL, (team.team_id,))), + referencing_users=tuple( + string_value(row["user_id"]) for row in read_rows(REFERENCING_USERS_SQL, (team.team_id,)) + ), + key_present=bool(read_rows(TOKEN_SQL, (team.hashed_key,))), + ) + + +def _states(fleet: Sequence[Team]) -> tuple[TeamState, ...]: + return tuple(_state(team) for team in fleet) + + +def _overrides() -> dict[str, str]: + return {"DATABASE_URL": os.environ["DATABASE_URL"]} + + +def _user(candidate: Gateway, scenario: Scenario) -> str: + """An internal user created through ``candidate``; its removal is registered on the shared rig.""" + user_id: Final = f"integration-chaos-{uuid.uuid4().hex}" + candidate.post("/user/new", {"user_id": user_id, "auto_create_key": False, "user_role": "internal_user"}) + scenario.cleanups.callback(scenario.delete_user, user_id) + return user_id + + +def _delete_team_if_present(candidate: Gateway, team_id: str) -> None: + if read_rows(TEAM_SQL, (team_id,)): + candidate.post("/team/delete", {"team_ids": [team_id]}) + assert read_rows(TEAM_SQL, (team_id,)) == [] + + +def _team(candidate: Gateway, scenario: Scenario, index: int) -> Team: + alias: Final = f"integration-chaos-{index:02d}-{uuid.uuid4().hex}" + team_id: Final = string_value(candidate.post("/team/new", {"team_alias": alias})["team_id"]) + scenario.cleanups.callback(_delete_team_if_present, scenario.gateway, team_id) + members: Final = tuple(_user(candidate, scenario) for _ in range(MEMBERS_PER_TEAM)) + candidate.post( + "/team/member_add", + {"team_id": team_id, "member": [{"role": "user", "user_id": user_id} for user_id in members]}, + ) + key: Final = string_value(candidate.post("/key/generate", {"team_id": team_id, "key_alias": alias})["key"]) + scenario.cleanups.callback(delete_key_if_present, scenario.gateway, key) + return Team(team_id, members, sha256(key.encode()).hexdigest()) + + +def _fleet(candidate: Gateway, scenario: Scenario) -> tuple[Team, ...]: + """24 teams created through ``candidate``, each verified intact: row, key, both members' membership + rows and ``teams`` entries present, so the invariants after the burst have something to remove. + + A master-key ``/team/new`` also seats ``default_user_id`` as an admin (roster entry, membership row and + ``teams`` entry), so the checks are supersets. Cleanup is registered on the shared rig; the team + callback only acts when a run fails before its delete. + """ + fleet: Final = tuple(_team(candidate, scenario, index) for index in range(TEAMS)) + for team, state in zip(fleet, _states(fleet)): + assert state.untouched, state.describe() + assert set(state.memberships) >= set(team.members), state.describe() + assert set(state.referencing_users) >= set(team.members), state.describe() + return fleet + + +class Burst: + """One ``/team/delete`` per team on ``target``, all submitted at once; ``chaos_point`` is set once the + third delete has answered (or failed), so the leg breaks the infrastructure mid-burst.""" + + def __init__(self, target: Gateway) -> None: + self._target: Final = target + self._lock: Final = threading.Lock() + self._answers = 0 # rebind-ok: counter behind _lock + self._futures: dict[str, Future[Outcome]] = {} + self.chaos_point: Final = threading.Event() + + def start(self, pool: ThreadPoolExecutor, fleet: Sequence[Team]) -> None: + assert not self._futures, "burst already started" + self._futures.update((team.team_id, pool.submit(self._delete, team)) for team in fleet) + assert self.chaos_point.wait(DELETE_TIMEOUT_SECONDS), ( + f"fewer than {CHAOS_AFTER_ANSWERS} deletes answered within {DELETE_TIMEOUT_SECONDS}s" + ) + + def _delete(self, team: Team) -> Outcome: + try: + response: Final = self._target.client.request( + "POST", + "/team/delete", + json={"team_ids": [team.team_id]}, + headers={"Authorization": f"Bearer {self._target.key}"}, + timeout=DELETE_TIMEOUT_SECONDS, + ) + outcome = Outcome(team.team_id, response.status_code, response.text[:200]) + except httpx.HTTPError as error: # a killed worker or a stopped proxy drops the in-flight request + outcome = Outcome(team.team_id, None, f"{type(error).__name__}: {error}"[:200]) + with self._lock: + self._answers += 1 + if self._answers >= CHAOS_AFTER_ANSWERS: + self.chaos_point.set() + return outcome + + def answered(self) -> int: + with self._lock: + return self._answers + + def pending(self, team_id: str) -> bool: + return not self._futures[team_id].done() + + def outcomes(self) -> tuple[Outcome, ...]: + return tuple(future.result(timeout=DELETE_TIMEOUT_SECONDS + 30) for future in self._futures.values()) + + +def _waiters_on_lock_held_by(backend_pid: int) -> int: + rows: Final = read_rows(WAITERS_ON_HELD_LOCK_SQL, ("%pg_advisory_xact_lock%", str(backend_pid))) + waiting: Final = rows[0]["waiting"] + assert isinstance(waiting, int) + return waiting + + +@contextmanager +def _holding_team_lock(team_id: str) -> Iterator[int]: + """Hold ``team_id``'s advisory lock in a test-owned transaction and yield the holder's backend pid; + leaving the block commits, which releases the lock.""" + with psycopg.connect(os.environ["DATABASE_URL"]) as holder: + holder.execute(TAKE_TEAM_LOCK_SQL, (team_id,)) + yield holder.info.backend_pid + + +def _await_pinned_delete_blocked(burst: Burst, pinned: Team, holder_pid: int, record_property: RecordProperty) -> None: + """The pinned team's delete is queued behind the held lock inside Postgres with its request unanswered, + so the failure applied next lands on a live delete; records how many other deletes had answered.""" + eventually(lambda: _waiters_on_lock_held_by(holder_pid), lambda waiting: waiting >= 1, seconds=20) + assert burst.pending(pinned.team_id), f"{pinned.team_id}: delete answered while its team lock was held" + record_property("answered_before_outage", burst.answered()) + + +def _record_burst( + record_property: RecordProperty, outcomes: Sequence[Outcome], observed: Sequence[TeamState], pinned: Team +) -> None: + """Record the status split, the pinned team's outcome and the half-deleted teams seen before the retry.""" + split: Final = Counter(outcome.label for outcome in outcomes) + record_property("status_split", dict(sorted(split.items()))) + pinned_outcome: Final = next(outcome for outcome in outcomes if outcome.team_id == pinned.team_id) + record_property( + "pinned_delete", + {"team_id": pinned.team_id, "status": pinned_outcome.status, "detail": pinned_outcome.detail}, + ) + record_property("partial_states_before_retry", [state.describe() for state in observed if state.partial]) + record_property("rows_present_before_retry", sum(state.row_present for state in observed)) + + +def _retry_survivors(target: Gateway, fleet: Sequence[Team], observed: Sequence[TeamState]) -> tuple[str, ...]: + """Delete once more, through ``target``, every team whose row survived the burst; each must answer 200.""" + survivors: Final = tuple(team.team_id for team, state in zip(fleet, observed) if state.row_present) + for team_id in survivors: + assert target.post("/team/delete", {"team_ids": [team_id]}) == {"deleted_teams": [team_id]} + return survivors + + +def _expected_tombstones(before: TeamState, retried: bool) -> int: + """One ``LiteLLM_DeletedTeamTable`` row per attempt that reached the tombstone write. + + Both legs commit the tombstone before the locked transaction that removes the team, so a burst + attempt that died in between left one (``before.tombstones``, 0 or 1) for a team whose row + survived, and the retry adds one more. Pinned as observed: pre-existing on the merge base, + outside this PR's diff, recorded in the audit report. + """ + assert before.tombstones <= 1, before.describe() + return before.tombstones + (1 if retried else 0) + + +def _assert_every_team_fully_deleted( + record_property: RecordProperty, + before_retry: Sequence[TeamState], + final: Sequence[TeamState], + retried: Sequence[str], +) -> None: + """Every team: row, memberships, ``teams`` references and key gone; tombstones one per attempt.""" + expected: Final = {state.team_id: _expected_tombstones(state, state.team_id in retried) for state in before_retry} + record_property("double_tombstones", sorted(team_id for team_id, count in expected.items() if count == 2)) + violations: Final = tuple( + f"{state.describe()} expected tombstones={expected[state.team_id]}" + for state in final + if not state.clean or state.tombstones != expected[state.team_id] or expected[state.team_id] == 0 + ) + assert not violations, ( + f"{len(violations)} of {len(final)} teams are not fully deleted after the retry:\n " + + "\n ".join(violations) + + f"\nhalf-deleted before the retry ({sum(state.partial for state in before_retry)}):\n " + + "\n ".join(state.describe() for state in before_retry if state.partial) + + f"\nretried ({len(retried)}): {sorted(retried)}" + ) + + +def _workers(root: psutil.Process) -> tuple[psutil.Process, ...]: + """uvicorn's worker children of the owned proxy root, spawned through ``multiprocessing.spawn``. + + The root's other child is the multiprocessing resource tracker; each worker's prisma query engine + is a grandchild. A worker that just died shows as a zombie whose cmdline raises, so it is left out. + """ + workers: Final = [] + for child in root.children(): + try: + cmdline = child.cmdline() + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + if any("multiprocessing.spawn" in part for part in cmdline): + workers.append(child) + return tuple(sorted(workers, key=lambda process: process.pid)) + + +def _cache_ping(target: Gateway) -> httpx.Response: + return target.request("GET", "/cache/ping") + + +def _cache_status(response: httpx.Response) -> str: + assert response.status_code == 200, f"/cache/ping: {response.status_code} {response.text}" + return string_value(JSON_OBJECT.validate_json(response.content)["status"]) + + +@pytest.mark.timeout(240) # owned two-worker proxy boot plus a 24-team fleet and its cleanup +def test_worker_killed_mid_burst_leaves_every_team_fully_deleted_after_retry( + gateway: Gateway, tmp_path: Path, record_property: RecordProperty +) -> None: + with ( + gateway.scenario() as scenario, + owned_proxy_process( + gateway, tmp_path, _overrides(), remove_environment=REMOVE_FROM_ENVIRONMENT, workers=WORKERS + ) as owned, + ThreadPoolExecutor(TEAMS) as pool, + ): + root: Final = psutil.Process(owned.process.pid) + fleet: Final = _fleet(owned.gateway, scenario) + pinned: Final = fleet[0] + burst: Final = Burst(owned.gateway) + with _holding_team_lock(pinned.team_id) as holder_pid: + burst.start(pool, fleet) + _await_pinned_delete_blocked(burst, pinned, holder_pid, record_property) + before: Final = _workers(root) + assert len(before) == WORKERS, [process.pid for process in before] + victim: Final = before[0] + victim.kill() # SIGKILL with the pinned delete blocked: the worker cannot finish its in-flight deletes + victim.wait(timeout=10) + with httpx.Client(base_url=str(owned.gateway.client.base_url), timeout=15, trust_env=False) as fresh: + readiness: Final = fresh.get("/health/readiness") + assert readiness.status_code == 200, ( + f"/health/readiness with worker {victim.pid} dead: {readiness.status_code} {readiness.text}" + ) + # The lock is released: the pinned delete finishes on the survivor, or was dropped with the victim. + outcomes: Final = burst.outcomes() + respawned: Final = eventually( + lambda: tuple(process.pid for process in _workers(root)), + lambda pids: len(pids) == WORKERS and victim.pid not in pids, + seconds=60, + ) + record_property( + "worker_pids", {"before": [process.pid for process in before], "killed": victim.pid, "after": respawned} + ) + observed: Final = _states(fleet) + _record_burst(record_property, outcomes, observed, pinned) + assert all(outcome.answered_or_dropped for outcome in outcomes), [ + (outcome.team_id, outcome.status, outcome.detail) for outcome in outcomes if not outcome.answered_or_dropped + ] + retried: Final = _retry_survivors(owned.gateway, fleet, observed) + _assert_every_team_fully_deleted(record_property, observed, _states(fleet), retried) + + +@pytest.mark.timeout(240) # owned Redis, owned two-worker proxy boot, 24-team fleet, Redis restart +def test_redis_stopped_mid_burst_keeps_deletes_answering_200( + gateway: Gateway, tmp_path: Path, record_property: RecordProperty +) -> None: + with ( + gateway.scenario() as scenario, + owned_redis(tmp_path) as coordination, + owned_proxy_process( + gateway, + tmp_path, + { + **_overrides(), + "REDIS_HOST": coordination.host, + "REDIS_PORT": str(coordination.port), + # The breaker opens during the outage; the default 60 s before it probes again would + # keep /cache/ping (whose set_cache runs under the breaker) at 503 long after restart. + "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT": "5", + }, + remove_environment=REMOVE_FROM_ENVIRONMENT, + workers=WORKERS, + ) as owned, + ThreadPoolExecutor(TEAMS) as pool, + ): + fleet: Final = _fleet(owned.gateway, scenario) + pinned: Final = fleet[0] + assert _cache_status(_cache_ping(owned.gateway)) == "healthy" + burst: Final = Burst(owned.gateway) + with _holding_team_lock(pinned.team_id) as holder_pid: + burst.start(pool, fleet) + _await_pinned_delete_blocked(burst, pinned, holder_pid, record_property) + coordination.stop() + down: Final = _cache_ping(owned.gateway) + assert down.status_code == 503, f"/cache/ping with Redis stopped: {down.status_code} {down.text}" + assert "Service Unhealthy" in down.text, down.text + assert burst.pending(pinned.team_id), f"{pinned.team_id}: delete answered while its team lock was held" + # The lock is released with Redis down: the pinned delete's cache eviction runs against the outage. + outcomes: Final = burst.outcomes() + coordination.start() + recovered: Final = eventually(lambda: _cache_ping(owned.gateway), lambda r: r.status_code == 200, seconds=60) + assert _cache_status(recovered) == "healthy" + + observed: Final = _states(fleet) + _record_burst(record_property, outcomes, observed, pinned) + assert all(outcome.status == 200 for outcome in outcomes), ( + "deletes not answered 200 while Redis was down: " + + str([(outcome.team_id, outcome.status, outcome.detail) for outcome in outcomes if outcome.status != 200]) + + f"; split {dict(Counter(outcome.label for outcome in outcomes))}" + ) + retried: Final = _retry_survivors(owned.gateway, fleet, observed) + _assert_every_team_fully_deleted(record_property, observed, _states(fleet), retried) + + +@pytest.mark.timeout(240) # two owned two-worker proxy boots (before and after SIGTERM) plus a 24-team fleet +def test_proxy_terminated_mid_burst_then_restarted_leaves_every_team_fully_deleted( + gateway: Gateway, tmp_path: Path, record_property: RecordProperty +) -> None: + with gateway.scenario() as scenario, ThreadPoolExecutor(TEAMS) as pool: + with owned_proxy_process( + gateway, tmp_path, _overrides(), remove_environment=REMOVE_FROM_ENVIRONMENT, workers=WORKERS + ) as doomed: + fleet: Final = _fleet(doomed.gateway, scenario) + pinned: Final = fleet[0] + burst: Final = Burst(doomed.gateway) + with _holding_team_lock(pinned.team_id) as holder_pid: + burst.start(pool, fleet) + _await_pinned_delete_blocked(burst, pinned, holder_pid, record_property) + doomed.process.terminate() # SIGTERM with the pinned delete blocked: uvicorn stops accepting and drains + assert burst.pending(pinned.team_id), f"{pinned.team_id}: delete answered while its team lock was held" + # The lock is released: the drain lets the pinned delete finish before the proxy exits. + doomed.process.wait(timeout=120) + outcomes: Final = burst.outcomes() + + at_restart: Final = _states(fleet) + _record_burst(record_property, outcomes, at_restart, pinned) + assert all(outcome.answered_or_dropped for outcome in outcomes), [ + (outcome.team_id, outcome.status, outcome.detail) for outcome in outcomes if not outcome.answered_or_dropped + ] + with owned_proxy_process( + gateway, tmp_path, _overrides(), remove_environment=REMOVE_FROM_ENVIRONMENT, workers=WORKERS + ) as fresh: + retried: Final = _retry_survivors(fresh.gateway, fleet, at_restart) + final: Final = _states(fleet) + _assert_every_team_fully_deleted(record_property, at_restart, final, retried) diff --git a/tests/integration/management/test_team_delete_inputs.py b/tests/integration/management/test_team_delete_inputs.py new file mode 100644 index 00000000000..d385c915696 --- /dev/null +++ b/tests/integration/management/test_team_delete_inputs.py @@ -0,0 +1,330 @@ +"""Sad inputs for /team/delete: malformed ids, callers without access, and rosters the API can no longer produce. + +Legacy roster shapes (email-only entries, entries with neither id nor email) are seeded straight into +``members_with_roles`` because ``/team/member_add`` backfills ``user_id`` and will not write them any more. +""" + +from __future__ import annotations + +import json +import os +import uuid +from collections.abc import Callable, Mapping +from hashlib import sha256 +from typing import Final + +import httpx +import pytest +from pydantic import JsonValue +from redis import Redis + +from tests.integration._support.client import ( + Gateway, + Scenario, + delete_key_if_present, + eventually, + object_value, + string_value, +) +from tests.integration._support.database import read_rows, write_rows + +RecordProperty = Callable[[str, object], None] + +TEAM_SQL: Final = 'SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s' +ROSTER_READ_SQL: Final = 'SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s' +ROSTER_SQL: Final = 'UPDATE "LiteLLM_TeamTable" SET members_with_roles = %s::jsonb WHERE team_id = %s' +MEMBERSHIP_SQL: Final = 'SELECT user_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s' +USER_SQL: Final = 'SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s' +USER_EMAIL_SQL: Final = 'UPDATE "LiteLLM_UserTable" SET user_email = %s WHERE user_id = %s' +TOKEN_SQL: Final = 'SELECT token FROM "LiteLLM_VerificationToken" WHERE token = %s' +TOMBSTONE_SQL: Final = 'SELECT id FROM "LiteLLM_DeletedTeamTable" WHERE team_id = %s' +AUDIT_SQL: Final = 'SELECT id, table_name, action FROM "LiteLLM_AuditLog" WHERE object_id = %s' + +NOT_FOUND: Final = "Team not found, passed team_id=" +# /team/delete sits on management_routes but on no internal-user route list, so the route gate in +# RouteChecks.non_proxy_admin_allowed_routes_check answers 401 before _verify_team_access ever runs +# (pinned by tests/integration/authorization/test_team_admin_gate.py as team_admin=401, others=401). +ROUTE_GATE_MESSAGE: Final = "Only proxy admin can be used" +UNKNOWN_TEAM: Final = f"integration-missing-{uuid.uuid4().hex}" +FIVE_KB_TEAM: Final = "t" * 5120 + + +def _team_rows(team_id: str) -> list[dict[str, JsonValue]]: + return read_rows(TEAM_SQL, (team_id,)) + + +def _delete_team_if_present(gateway: Gateway, team_id: str) -> None: + """Cleanup for a team the test deletes itself: a no-op once the row is gone.""" + if _team_rows(team_id): + gateway.post("/team/delete", {"team_ids": [team_id]}) + assert _team_rows(team_id) == [] + + +def _reset_roster_if_present(team_id: str) -> None: + """Cleanup for a seeded roster: put back a shape the delete path always accepts.""" + if _team_rows(team_id): + write_rows(ROSTER_SQL, ("[]", team_id)) + + +def _delete_user_if_present(gateway: Gateway, user_id: str) -> None: + if read_rows(USER_SQL, (user_id,)): + response: Final = gateway.request("POST", "/user/delete", {"user_ids": [user_id]}) + assert response.status_code == 200, response.text + assert read_rows(USER_SQL, (user_id,)) == [] + + +def _own_team(scenario: Scenario, **fields: JsonValue) -> str: + """A team the test deletes itself, so cleanup tolerates the row already being gone.""" + created: Final = scenario.gateway.post("/team/new", {"team_alias": f"integration-{uuid.uuid4().hex}", **fields}) + team_id: Final = string_value(created["team_id"]) + scenario.cleanups.callback(_delete_team_if_present, scenario.gateway, team_id) + return team_id + + +def _own_key(scenario: Scenario, **fields: JsonValue) -> str: + """A key the team delete is expected to remove, so cleanup tolerates it already being gone.""" + created: Final = scenario.gateway.post("/key/generate", fields) + token: Final = string_value(created["key"]) + scenario.cleanups.callback(delete_key_if_present, scenario.gateway, token) + return token + + +def _own_user(scenario: Scenario) -> str: + """An internal user the test may remove by SQL, so cleanup tolerates the row already being gone.""" + created: Final = scenario.gateway.post( + "/user/new", + {"user_id": f"integration-{uuid.uuid4().hex}", "auto_create_key": False, "user_role": "internal_user"}, + ) + user_id: Final = string_value(created["user_id"]) + scenario.cleanups.callback(_delete_user_if_present, scenario.gateway, user_id) + return user_id + + +def _seed_roster(scenario: Scenario, team_id: str, entries: list[dict[str, JsonValue]]) -> None: + write_rows(ROSTER_SQL, (json.dumps(entries), team_id)) + scenario.cleanups.callback(_reset_roster_if_present, team_id) + + +def _team_admin(scenario: Scenario, team_id: str) -> str: + """Add a member and flip their roster role to admin by SQL: the API gates that role behind a license.""" + user_id: Final = scenario.member(team_id) + rows: Final = read_rows(ROSTER_READ_SQL, (team_id,)) + assert len(rows) == 1, rows + roster: Final = rows[0]["members_with_roles"] + assert isinstance(roster, list), roster + promoted: Final = [ + {**object_value(entry), "role": "admin"} if object_value(entry).get("user_id") == user_id else entry + for entry in roster + ] + assert any(object_value(entry).get("user_id") == user_id for entry in promoted), promoted + write_rows(ROSTER_SQL, (json.dumps(promoted), team_id)) + return user_id + + +def _membership_user_ids(team_id: str) -> frozenset[str]: + return frozenset(string_value(row["user_id"]) for row in read_rows(MEMBERSHIP_SQL, (team_id,))) + + +def _delete(gateway: Gateway, team_ids: JsonValue, *, key: str | None = None) -> httpx.Response: + return gateway.request("POST", "/team/delete", {"team_ids": team_ids}, key=key) + + +def _hashed(token: str) -> str: + return sha256(token.encode()).hexdigest() + + +@pytest.mark.parametrize( + ("body", "status", "needle"), + [ + pytest.param({"team_ids": [UNKNOWN_TEAM]}, 404, f"{NOT_FOUND}{UNKNOWN_TEAM}", id="S1-unknown-id"), + pytest.param({"team_ids": "abc"}, 422, "list_type", id="S3-string-not-list"), + pytest.param({"team_ids": [123]}, 422, "string_type", id="S4-integer-item"), + pytest.param({"team_ids": [""]}, 404, NOT_FOUND, id="S5-empty-id"), + pytest.param({"team_ids": [FIVE_KB_TEAM]}, 404, NOT_FOUND, id="S6-5kb-id"), + ], +) +def test_rejects_malformed_team_ids(gateway: Gateway, body: Mapping[str, JsonValue], status: int, needle: str) -> None: + response: Final = gateway.request("POST", "/team/delete", body) + assert response.status_code == status, f"{response.status_code} {response.text}" + assert needle in response.text, response.text + + +def test_empty_list_deletes_nothing(gateway: Gateway) -> None: + response: Final = _delete(gateway, []) + assert response.status_code == 200, f"{response.status_code} {response.text}" + assert response.json() == {"deleted_teams": []}, response.text + + +def test_duplicate_ids_delete_once(gateway: Gateway, record_property: RecordProperty) -> None: + """Repeated ids collapse to one delete: the body names the team once and exactly one tombstone row lands.""" + with gateway.scenario() as scenario: + team: Final = _own_team(scenario) + first: Final = scenario.member(team) + second: Final = scenario.member(team) + key: Final = _own_key(scenario, team_id=team) + # The master key's /team/new also seats the proxy admin, so the table holds more than these two. + assert {first, second} <= _membership_user_ids(team), _membership_user_ids(team) + response: Final = _delete(gateway, [team, team]) + # Read every table before the first assert so a red cell carries the partial state with it. + present: Final = _team_rows(team) + memberships: Final = _membership_user_ids(team) + key_rows: Final = read_rows(TOKEN_SQL, (_hashed(key),)) + tombstones: Final = read_rows(TOMBSTONE_SQL, (team,)) + audit: Final = read_rows(AUDIT_SQL, (team,)) + state: Final = ( + f"team_present={bool(present)} membership_rows={len(memberships)} key_present={bool(key_rows)} " + f"tombstone_rows={len(tombstones)} audit_rows={len(audit)}" + ) + record_property("status", response.status_code) + record_property("body", response.text) + record_property("state_after", state) + record_property("audit_rows", len(audit)) # recorded only: the shared rigs cannot enable audit logging + assert response.status_code == 200, f"{response.status_code} {response.text}; {state}" + assert response.json() == {"deleted_teams": [team]}, response.text + assert present == [], state + assert memberships == frozenset(), state + assert key_rows == [], state + assert len(tombstones) == 1, f"tombstone rows for {team}: {len(tombstones)}; {state}" + + +def test_missing_authorization_is_401(gateway: Gateway) -> None: + response: Final = gateway.client.post("/team/delete", json={"team_ids": [UNKNOWN_TEAM]}) + assert response.status_code == 401, f"{response.status_code} {response.text}" + assert "error" in response.text.lower(), response.text + + +def test_internal_user_outside_team_is_refused_by_the_route_gate(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team() + outsider: Final = scenario.user(user_role="internal_user") + key: Final = scenario.key(user_id=outsider) + response: Final = _delete(gateway, [team], key=key) + assert response.status_code == 401, f"{response.status_code} {response.text}" + assert ROUTE_GATE_MESSAGE in response.text, response.text + assert len(_team_rows(team)) == 1 + + +def test_admin_of_another_team_is_refused_by_the_route_gate(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + target: Final = scenario.team() + other: Final = scenario.team() + admin: Final = _team_admin(scenario, other) + key: Final = scenario.key(team_id=other, user_id=admin) + response: Final = _delete(gateway, [target], key=key) + assert response.status_code == 401, f"{response.status_code} {response.text}" + assert ROUTE_GATE_MESSAGE in response.text, response.text + assert len(_team_rows(target)) == 1 + assert len(_team_rows(other)) == 1 + + +def test_team_admin_of_own_team_is_refused_by_the_route_gate(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team() + admin: Final = _team_admin(scenario, team) + key: Final = scenario.key(team_id=team, user_id=admin) + response: Final = _delete(gateway, [team], key=key) + assert response.status_code == 401, f"{response.status_code} {response.text}" + assert ROUTE_GATE_MESSAGE in response.text, response.text + assert len(_team_rows(team)) == 1 + + +def test_roster_user_whose_row_was_removed(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = _own_team(scenario) + ghost: Final = _own_user(scenario) + gateway.post("/team/member_add", {"team_id": team, "member": {"role": "user", "user_id": ghost}}) + assert ghost in _membership_user_ids(team), _membership_user_ids(team) + write_rows('DELETE FROM "LiteLLM_UserTable" WHERE user_id = %s', (ghost,)) + assert read_rows(USER_SQL, (ghost,)) == [] + response: Final = _delete(gateway, [team]) + assert response.status_code == 200, f"{response.status_code} {response.text}" + assert _team_rows(team) == [] + assert read_rows(MEMBERSHIP_SQL, (team,)) == [] + + +def test_email_only_roster_entry_matching_no_user(gateway: Gateway, record_property: RecordProperty) -> None: + with gateway.scenario() as scenario: + team: Final = _own_team(scenario) + _seed_roster( + scenario, + team, + [{"role": "user", "user_id": None, "user_email": f"nobody-{uuid.uuid4().hex}@example.com"}], + ) + response: Final = _delete(gateway, [team]) + record_property("status", response.status_code) + record_property("body", response.text) + assert response.status_code == 200, f"{response.status_code} {response.text}" + assert _team_rows(team) == [] + + +def test_email_only_roster_entry_matching_two_case_variants(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + tag: Final = uuid.uuid4().hex + upper: Final = scenario.user(user_role="internal_user") + lower: Final = scenario.user(user_role="internal_user") + # /user/new rejects a second email that matches case-insensitively, so the pair is seeded by SQL. + write_rows(USER_EMAIL_SQL, (f"Case-{tag}@example.com", upper)) + write_rows(USER_EMAIL_SQL, (f"case-{tag}@example.com", lower)) + for user in (upper, lower): + gateway.chat(model, key=scenario.key(user_id=user, models=[model])) + with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache: + + def cached() -> dict[str, int]: + return {user: int(cache.exists(user)) for user in (upper, lower)} + + assert eventually(cached, lambda seen: seen == {upper: 1, lower: 1}, seconds=10) == {upper: 1, lower: 1} + team: Final = _own_team(scenario) + _seed_roster(scenario, team, [{"role": "user", "user_id": None, "user_email": f"CASE-{tag}@EXAMPLE.COM"}]) + response: Final = _delete(gateway, [team]) + assert response.status_code == 200, f"{response.status_code} {response.text}" + assert _team_rows(team) == [] + remaining: Final = eventually( + cached, lambda seen: seen == {upper: 0, lower: 0}, seconds=10, return_last_on_timeout=True + ) + assert remaining == {upper: 0, lower: 0}, ( + f"user cache entries still present after /team/delete: " + f"{upper} exists={remaining[upper]}, {lower} exists={remaining[lower]}" + ) + + +def test_roster_entry_without_id_or_email_pins_the_500(gateway: Gateway, record_property: RecordProperty) -> None: + """Pins a pre-existing defect outside this PR's diff until it gets its own ticket: for a roster entry with neither + id nor email, LiteLLM_TeamTable.model_validate raises outside delete_team's 404 try/except, so the call is a 500 + that writes nothing (team row intact, no tombstone, membership rows untouched).""" + with gateway.scenario() as scenario: + team: Final = _own_team(scenario) + before: Final = _membership_user_ids(team) + _seed_roster(scenario, team, [{"role": "user", "user_id": None, "user_email": None}]) + response: Final = _delete(gateway, [team]) + present: Final = _team_rows(team) + tombstones: Final = read_rows(TOMBSTONE_SQL, (team,)) + after: Final = _membership_user_ids(team) + state: Final = f"team_present={bool(present)} tombstone_rows={len(tombstones)} membership_rows={len(after)}" + record_property("status", response.status_code) + record_property("body", response.text) + record_property("state_after", state) + assert response.status_code == 500, f"{response.status_code} {response.text}; {state}" + assert "Internal server error" in response.text, response.text + assert len(present) == 1, state + assert tombstones == [], state + assert after == before, f"membership rows changed: before={sorted(before)} after={sorted(after)}" + + +def test_failed_delete_leaves_unrelated_key_serving(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + key: Final = scenario.key(models=[model]) + assert object_value(gateway.chat(model, key=key)["usage"])["total_tokens"] == 40 + missing: Final = f"integration-missing-{uuid.uuid4().hex}" + response: Final = _delete(gateway, [missing]) + assert response.status_code == 404, f"{response.status_code} {response.text}" + assert f"{NOT_FOUND}{missing}" in response.text, response.text + completion: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "after failed delete"}]}, + key=key, + ) + assert completion.status_code == 200, f"{completion.status_code} {completion.text}" + assert object_value(object_value(completion.json())["usage"])["total_tokens"] == 40 diff --git a/tests/integration/management/test_team_delete_large_membership.py b/tests/integration/management/test_team_delete_large_membership.py new file mode 100644 index 00000000000..5912256e5cd --- /dev/null +++ b/tests/integration/management/test_team_delete_large_membership.py @@ -0,0 +1,634 @@ +"""`/team/delete` as one locked transaction, whatever the roster size. + +The delete removes the team row, its membership rows, every member's `teams` reference and every +team key in one pass, writes one tombstone per team, evicts the cached team object and takes the +team's advisory lock (the one `/team/member_add` takes) before it writes. A roster larger than the +Prisma pool used to fail with P2028 because each member got its own transaction. +""" + +from __future__ import annotations + +import json +import os +import uuid +from collections.abc import Callable, Mapping, Sequence +from concurrent.futures import Future, ThreadPoolExecutor +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import httpx +import psycopg +import pytest +import yaml +from pydantic import JsonValue +from redis import Redis + +from tests.integration._support.client import ( + Gateway, + Scenario, + delete_key_if_present, + eventually, + object_value, + string_value, +) +from tests.integration._support.database import read_rows, write_rows +from tests.integration._support.process import owned_proxy_process + +LARGE_ROSTER: Final = 250 +POOL_LIMIT: Final = 5 +# One statement seeds the whole roster: 250 individual /user/new calls would dominate the runtime. +SEED_USERS_SQL: Final = """ +INSERT INTO "LiteLLM_UserTable" (user_id, user_role, teams, models) +SELECT %s || '-' || lpad(n::text, 3, '0'), 'internal_user', '{}'::text[], '{}'::text[] +FROM generate_series(1, %s::int) AS n +""" +# The master key's user id. `/team/new` appends the creator to the roster as an admin, so every team +# created here carries this member alongside the ones the test adds. +PROXY_ADMIN: Final = "default_user_id" +TAKE_TEAM_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext(%s))" +# Sessions blocked on the advisory lock the given backend holds, and nothing else on the shared rig. +WAITERS_ON_HELD_LOCK_SQL: Final = """ +SELECT count(*)::int AS waiting +FROM pg_locks waiter +JOIN pg_stat_activity session ON session.pid = waiter.pid +WHERE waiter.locktype = 'advisory' + AND NOT waiter.granted + AND session.wait_event_type = 'Lock' + AND session.query ILIKE %s + AND (waiter.classid, waiter.objid, waiter.objsubid) IN ( + SELECT held.classid, held.objid, held.objsubid + FROM pg_locks held + WHERE held.locktype = 'advisory' AND held.granted AND held.pid = %s::int + ) +""" + + +def _hashed(key: str) -> str: + return sha256(key.encode()).hexdigest() + + +def _team_rows(team_id: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)) + + +def _membership_user_ids(team_id: str) -> list[JsonValue]: + rows: Final = read_rows( + 'SELECT user_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s ORDER BY user_id', (team_id,) + ) + return [row["user_id"] for row in rows] + + +def _user_teams(user_id: str) -> JsonValue: + rows: Final = read_rows('SELECT teams FROM "LiteLLM_UserTable" WHERE user_id = %s', (user_id,)) + assert len(rows) == 1, f"user row for {user_id}: {rows}" + return rows[0]["teams"] + + +def _users_referencing(team_id: str) -> list[JsonValue]: + rows: Final = read_rows( + 'SELECT user_id FROM "LiteLLM_UserTable" WHERE %s = ANY(teams) ORDER BY user_id', (team_id,) + ) + return [row["user_id"] for row in rows] + + +def _user_ids_with_prefix(prefix: str) -> list[JsonValue]: + rows: Final = read_rows( + 'SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id LIKE %s ORDER BY user_id', (f"{prefix}-%",) + ) + return [row["user_id"] for row in rows] + + +def _live_token(hashed: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT token, team_id FROM "LiteLLM_VerificationToken" WHERE token = %s', (hashed,)) + + +def _deleted_token(hashed: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT token, team_id FROM "LiteLLM_DeletedVerificationToken" WHERE token = %s', (hashed,)) + + +def _tombstones(team_id: str) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT team_id, members_with_roles FROM "LiteLLM_DeletedTeamTable" WHERE team_id = %s', (team_id,) + ) + + +def _roster_user_ids(roster: JsonValue) -> list[str]: + assert isinstance(roster, list), f"roster is not a list: {roster!r}" + return sorted(string_value(object_value(member)["user_id"]) for member in roster) + + +def _waiters_on_lock_held_by(backend_pid: int) -> int: + rows: Final = read_rows(WAITERS_ON_HELD_LOCK_SQL, ("%pg_advisory_xact_lock%", str(backend_pid))) + waiting: Final = rows[0]["waiting"] + assert isinstance(waiting, int) + return waiting + + +def _remove_team_by_sql(team_id: str) -> None: + """Cleanup for a team the test expects to have deleted itself. Whatever a failed delete left behind + (row, memberships, `teams` references) goes by SQL so the shared rig stays clean without sending + another request through the proxy under test.""" + if not _team_rows(team_id): + return + write_rows('DELETE FROM "LiteLLM_TeamMembership" WHERE team_id = %s', (team_id,)) + write_rows( + 'UPDATE "LiteLLM_UserTable" SET teams = array_remove(teams, %s) WHERE %s = ANY(teams)', (team_id, team_id) + ) + write_rows('DELETE FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)) + assert _team_rows(team_id) == [] + + +def _remove_team_if_present(gateway: Gateway, team_id: str) -> None: + """Cleanup for teams the test deletes itself: the API delete first, SQL for anything it leaves.""" + if not _team_rows(team_id): + return + gateway.request("POST", "/team/delete", {"team_ids": [team_id]}) + _remove_team_by_sql(team_id) + + +def _create_team(scenario: Scenario, **fields: JsonValue) -> str: + """A team the test deletes itself; cleanup only removes it if the test left it behind.""" + created: Final = scenario.gateway.post("/team/new", {"team_alias": f"integration-{uuid.uuid4().hex}", **fields}) + team_id: Final = string_value(created["team_id"]) + scenario.cleanups.callback(_remove_team_if_present, scenario.gateway, team_id) + return team_id + + +def _generate_key(scenario: Scenario, **fields: JsonValue) -> str: + """A key the team delete is expected to remove; cleanup only deletes it if it is still live.""" + key: Final = string_value(scenario.gateway.post("/key/generate", fields)["key"]) + scenario.cleanups.callback(delete_key_if_present, scenario.gateway, key) + return key + + +def _delete_seeded_users(prefix: str) -> None: + write_rows('DELETE FROM "LiteLLM_UserTable" WHERE user_id LIKE %s', (f"{prefix}-%",)) + assert _user_ids_with_prefix(prefix) == [] + + +def _seed_users(scenario: Scenario, prefix: str, count: int) -> tuple[str, ...]: + """Insert `count` user rows in one statement; ids are `-001` … `-`.""" + users: Final = tuple(f"{prefix}-{index:03d}" for index in range(1, count + 1)) + write_rows(SEED_USERS_SQL, (prefix, str(count))) + scenario.cleanups.callback(_delete_seeded_users, prefix) + assert _user_ids_with_prefix(prefix) == list(users) + return users + + +def _bulk_member_add(gateway: Gateway, team_id: str, users: Sequence[str]) -> None: + gateway.post( + "/team/member_add", + {"team_id": team_id, "member": [{"role": "user", "user_id": user_id} for user_id in users]}, + ) + + +def _delete_teams(gateway: Gateway, team_ids: Sequence[str]) -> httpx.Response: + return gateway.request("POST", "/team/delete", {"team_ids": list(team_ids)}) + + +def _team_info(gateway: Gateway, team_id: str) -> httpx.Response: + return gateway.request("GET", "/team/info", params={"team_id": team_id}) + + +def _team_not_found_body(team_id: str) -> dict[str, JsonValue]: + """The proxy's exception handler wraps the 404 detail as an `error` object with the detail stringified.""" + return { + "error": { + "message": f"{{'message': 'Team not found, passed team id: {team_id}.'}}", + "type": "auth_error", + "param": "None", + "code": "404", + } + } + + +def _chat(gateway: Gateway, model: str, key: str) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"team delete {uuid.uuid4().hex}"}]}, + key=key, + ) + + +def _post_with_timeout(gateway: Gateway, path: str, body: Mapping[str, JsonValue], timeout: float) -> httpx.Response: + """Like `Gateway.request` with a per-call timeout longer than the client's default 15 s.""" + return gateway.client.request( + "POST", path, json=body, headers={"Authorization": f"Bearer {gateway.key}"}, timeout=timeout + ) + + +def _post_in_background( + pool: ThreadPoolExecutor, gateway: Gateway, path: str, body: Mapping[str, JsonValue] +) -> Future[httpx.Response]: + return pool.submit(_post_with_timeout, gateway, path, body, 60) + + +def _config_with_pool_limit(tmp_path: Path, pool_limit: int) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["general_settings"]["database_connection_pool_limit"] = pool_limit + config["general_settings"]["database_connection_pool_timeout"] = 60 + path: Final = tmp_path / f"pool-{pool_limit}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _redis() -> Redis: + return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) + + +def test_delete_small_team_removes_rows_keys_tombstone_and_cache( + gateway: Gateway, record_property: Callable[[str, object], None] +) -> None: + with gateway.scenario() as scenario, _redis() as cache: + model: Final = scenario.model() + users: Final = sorted(scenario.user() for _ in range(3)) + team: Final = _create_team(scenario) + _bulk_member_add(gateway, team, users) + keys: Final = tuple(_generate_key(scenario, team_id=team) for _ in range(2)) + hashed: Final = tuple(_hashed(key) for key in keys) + roster: Final = sorted([PROXY_ADMIN, *users]) + assert _membership_user_ids(team) == roster + assert _users_referencing(team) == roster + assert all(_user_teams(user) == [team] for user in users), [_user_teams(user) for user in users] + assert all(len(_live_token(digest)) == 1 for digest in hashed), hashed + + warm: Final = _chat(gateway, model, keys[0]) + assert warm.status_code == 200, warm.text + team_cache_key: Final = f"team_id:{team}" + eventually(lambda: cache.exists(team_cache_key), lambda present: present == 1, seconds=10) + record_property("redis_keys_before_delete", sorted(entry.decode() for entry in cache.keys(f"*{team}*"))) + + response: Final = _delete_teams(gateway, [team]) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _users_referencing(team) == [] + assert [_user_teams(user) for user in users] == [[], [], []] + assert [_live_token(digest) for digest in hashed] == [[], []] + assert [_deleted_token(digest) for digest in hashed] == [ + [{"token": hashed[0], "team_id": team}], + [{"token": hashed[1], "team_id": team}], + ] + tombstones: Final = _tombstones(team) + assert len(tombstones) == 1, tombstones + assert tombstones[0]["team_id"] == team + assert _roster_user_ids(tombstones[0]["members_with_roles"]) == roster + + info: Final = _team_info(gateway, team) + assert info.status_code == 404, info.text + assert info.json() == _team_not_found_body(team) + + assert cache.exists(team_cache_key) == 0 + record_property("redis_keys_after_delete", sorted(entry.decode() for entry in cache.keys(f"*{team}*"))) + + +@pytest.mark.timeout(240) # owned two-worker proxy boot plus a 250-member roster +def test_delete_250_member_team_succeeds_with_pool_limit_five_on_two_workers( + gateway: Gateway, tmp_path: Path, record_property: Callable[[str, object], None] +) -> None: + prefix: Final = f"integration-roster-{uuid.uuid4().hex}" + # The scenario is bound to the shared gateway and its cleanups are SQL, so a failed delete on the + # owned proxy (and whatever it does to that proxy's workers) cannot mask the assertion below with a + # second failure during cleanup. The owned proxy is stopped before the cleanups run. + with ( + gateway.scenario() as scenario, + owned_proxy_process( + gateway, + tmp_path, + {"DATABASE_URL": os.environ["DATABASE_URL"]}, + config=_config_with_pool_limit(tmp_path, POOL_LIMIT), + remove_environment=("DATABASE_URL_READ_REPLICA",), + workers=2, + ) as owned, + ): + team: Final = string_value( + owned.gateway.post("/team/new", {"team_alias": f"integration-{uuid.uuid4().hex}"})["team_id"] + ) + scenario.cleanups.callback(_remove_team_by_sql, team) + users: Final = _seed_users(scenario, prefix, LARGE_ROSTER) + added: Final = _post_with_timeout( + owned.gateway, + "/team/member_add", + {"team_id": team, "member": [{"role": "user", "user_id": user_id} for user_id in users]}, + timeout=120, + ) + assert added.status_code == 200, added.text + roster: Final = sorted([PROXY_ADMIN, *users]) + assert _membership_user_ids(team) == roster + assert _users_referencing(team) == roster + + response: Final = _post_with_timeout(owned.gateway, "/team/delete", {"team_ids": [team]}, timeout=120) + record_property("h2_delete_response", f"{response.status_code} {response.text[:300]}") + assert response.status_code == 200, ( + f"/team/delete of a {LARGE_ROSTER}-member team with database_connection_pool_limit={POOL_LIMIT}: " + f"{response.status_code} {response.text}" + ) + assert response.json() == {"deleted_teams": [team]} + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _users_referencing(team) == [] + assert _user_ids_with_prefix(prefix) == list(users) + tombstones: Final = _tombstones(team) + assert len(tombstones) == 1, tombstones + assert _roster_user_ids(tombstones[0]["members_with_roles"]) == roster + + +def test_delete_waits_for_the_team_advisory_lock_and_completes_after_release( + gateway: Gateway, record_property: Callable[[str, object], None] +) -> None: + with gateway.scenario() as scenario: + user: Final = scenario.user() + team: Final = _create_team(scenario) + _bulk_member_add(gateway, team, [user]) + hashed: Final = _hashed(_generate_key(scenario, team_id=team)) + with ThreadPoolExecutor(max_workers=1) as pool: + with psycopg.connect(os.environ["DATABASE_URL"]) as holder: + holder.execute(TAKE_TEAM_LOCK_SQL, (team,)) + pending: Final = _post_in_background(pool, gateway, "/team/delete", {"team_ids": [team]}) + eventually( + lambda: _waiters_on_lock_held_by(holder.info.backend_pid), + lambda waiting: waiting >= 1, + seconds=20, + ) + assert not pending.done(), "delete returned while the team lock was still held" + assert _team_rows(team) == [{"team_id": team}], "team row deleted while the team lock was held" + # Recorded before the count assertion so both legs document what the delete had already + # written by the time it reached the lock. + record_property( + "state_while_blocked", + json.dumps( + { + "membership_user_ids": _membership_user_ids(team), + "user_teams": _user_teams(user), + "live_token_rows": len(_live_token(hashed)), + "tombstones": len(_tombstones(team)), + } + ), + ) + # One transaction per delete: a per-member fan-out would queue one waiter per roster entry. + eventually( + lambda: _waiters_on_lock_held_by(holder.info.backend_pid), + lambda waiting: waiting == 1, + seconds=10, + ) + # leaving the holder block commits its transaction, which releases the advisory lock + response: Final = pending.result(timeout=60) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _user_teams(user) == [] + assert _live_token(hashed) == [] + assert len(_tombstones(team)) == 1, _tombstones(team) + + +def test_deleting_two_teams_sharing_a_member_in_one_call_clears_both_from_the_member(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + user: Final = scenario.user() + first: Final = _create_team(scenario) + second: Final = _create_team(scenario) + _bulk_member_add(gateway, first, [user]) + _bulk_member_add(gateway, second, [user]) + assert _user_teams(user) == [first, second] + + response: Final = _delete_teams(gateway, [first, second]) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [first, second]} + + assert _team_rows(first) == [] + assert _team_rows(second) == [] + assert _membership_user_ids(first) == [] + assert _membership_user_ids(second) == [] + assert _user_teams(user) == [] + assert [row["team_id"] for row in _tombstones(first)] == [first] + assert [row["team_id"] for row in _tombstones(second)] == [second] + + +def test_deleting_one_team_leaves_the_members_other_team_and_key_intact(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + user: Final = scenario.user() + deleted: Final = _create_team(scenario) + kept: Final = scenario.team() + _bulk_member_add(gateway, deleted, [user]) + _bulk_member_add(gateway, kept, [user]) + kept_key: Final = scenario.key(team_id=kept, user_id=user) + before: Final = _chat(gateway, model, kept_key) + assert before.status_code == 200, before.text + + response: Final = _delete_teams(gateway, [deleted]) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [deleted]} + + assert _team_rows(deleted) == [] + assert _team_rows(kept) == [{"team_id": kept}] + assert _membership_user_ids(deleted) == [] + assert _membership_user_ids(kept) == [PROXY_ADMIN, user] + assert _user_teams(user) == [kept] + assert _live_token(_hashed(kept_key)) == [{"token": _hashed(kept_key), "team_id": kept}] + after: Final = _chat(gateway, model, kept_key) + assert after.status_code == 200, after.text + + +@pytest.mark.timeout(240) # owned proxy boot +def test_delete_writes_one_audit_row_for_the_team_and_one_per_key(gateway: Gateway, tmp_path: Path) -> None: + with ( + owned_proxy_process( + gateway, + tmp_path, + {"DATABASE_URL": os.environ["DATABASE_URL"], "LITELLM_STORE_AUDIT_LOGS": "true"}, + remove_environment=("DATABASE_URL_READ_REPLICA",), + ) as owned, + owned.gateway.scenario() as scenario, + ): + user: Final = scenario.user() + team: Final = _create_team(scenario) + _bulk_member_add(owned.gateway, team, [user]) + hashed: Final = _hashed(_generate_key(scenario, team_id=team)) + + response: Final = _delete_teams(owned.gateway, [team]) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + assert _team_rows(team) == [] + + def deleted_audit_rows() -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT table_name, action, object_id FROM "LiteLLM_AuditLog" ' + "WHERE object_id IN (%s, %s) AND action = 'deleted' ORDER BY table_name", + (team, hashed), + ) + + rows: Final = eventually(deleted_audit_rows, lambda found: len(found) >= 2, seconds=30) + assert rows == [ + {"table_name": "LiteLLM_TeamTable", "action": "deleted", "object_id": team}, + {"table_name": "LiteLLM_VerificationToken", "action": "deleted", "object_id": hashed}, + ] + + +def test_second_delete_of_the_same_team_is_404_with_one_tombstone(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + user: Final = scenario.user() + team: Final = _create_team(scenario) + _bulk_member_add(gateway, team, [user]) + + first: Final = _delete_teams(gateway, [team]) + assert first.status_code == 200, first.text + assert first.json() == {"deleted_teams": [team]} + + second: Final = _delete_teams(gateway, [team]) + assert second.status_code == 404, second.text + assert second.json() == {"detail": {"error": f"Team not found, passed team_id={team}"}} + + assert _team_rows(team) == [] + assert [row["team_id"] for row in _tombstones(team)] == [team] + assert _user_teams(user) == [] + + +def test_delete_empty_team_writes_tombstone_and_team_info_is_404(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = _create_team(scenario) + present: Final = _team_info(gateway, team) + assert present.status_code == 200, present.text + + response: Final = _delete_teams(gateway, [team]) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _tombstones(team) == [ + {"team_id": team, "members_with_roles": [{"role": "admin", "user_id": PROXY_ADMIN, "user_email": None}]} + ] + info: Final = _team_info(gateway, team) + assert info.status_code == 404, info.text + assert info.json() == _team_not_found_body(team) + + +def test_delete_keys_only_team_removes_keys_and_revokes_them(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + team: Final = _create_team(scenario) + keys: Final = tuple(_generate_key(scenario, team_id=team) for _ in range(2)) + hashed: Final = tuple(_hashed(key) for key in keys) + assert _membership_user_ids(team) == [PROXY_ADMIN] + for key in keys: + warm = _chat(gateway, model, key) + assert warm.status_code == 200, warm.text + + response: Final = _delete_teams(gateway, [team]) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": [team]} + + assert _team_rows(team) == [] + assert [_live_token(digest) for digest in hashed] == [[], []] + assert [_deleted_token(digest) for digest in hashed] == [ + [{"token": hashed[0], "team_id": team}], + [{"token": hashed[1], "team_id": team}], + ] + for key in keys: + revoked = _chat(gateway, model, key) + assert revoked.status_code == 401, f"{revoked.status_code} {revoked.text}" + + +def test_delete_three_teams_in_one_call_lists_all_and_tombstones_each_once(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + teams: Final = tuple(_create_team(scenario) for _ in range(3)) + for team in teams: + _bulk_member_add(gateway, team, [scenario.user()]) + + response: Final = _delete_teams(gateway, teams) + assert response.status_code == 200, response.text + assert response.json() == {"deleted_teams": list(teams)} + + for team in teams: + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _users_referencing(team) == [] + assert [row["team_id"] for row in _tombstones(team)] == [team] + + +def test_recreating_the_same_team_id_after_delete_serves_the_fresh_team(gateway: Gateway) -> None: + with gateway.scenario() as scenario, _redis() as cache: + model: Final = scenario.model() + original_member: Final = scenario.user() + replacement_member: Final = scenario.user() + team: Final = _create_team(scenario) + _bulk_member_add(gateway, team, [original_member]) + original_key: Final = _generate_key(scenario, team_id=team) + warm: Final = _chat(gateway, model, original_key) + assert warm.status_code == 200, warm.text + team_cache_key: Final = f"team_id:{team}" + eventually(lambda: cache.exists(team_cache_key), lambda present: present == 1, seconds=10) + + response: Final = _delete_teams(gateway, [team]) + assert response.status_code == 200, response.text + assert _team_rows(team) == [] + assert cache.exists(team_cache_key) == 0 + + fresh_alias: Final = f"integration-recreated-{uuid.uuid4().hex}" + recreated: Final = gateway.request( + "POST", + "/team/new", + { + "team_id": team, + "team_alias": fresh_alias, + "members_with_roles": [{"role": "user", "user_id": replacement_member}], + }, + ) + assert recreated.status_code == 200, recreated.text + assert recreated.json()["team_id"] == team + + info: Final = _team_info(gateway, team) + assert info.status_code == 200, info.text + team_info: Final = object_value(info.json()["team_info"]) + assert team_info["team_alias"] == fresh_alias + assert _roster_user_ids(team_info["members_with_roles"]) == [PROXY_ADMIN, replacement_member] + assert _membership_user_ids(team) == [PROXY_ADMIN, replacement_member] + assert _user_teams(replacement_member) == [team] + assert _user_teams(original_member) == [] + + fresh_key: Final = scenario.key(team_id=team) + served: Final = _chat(gateway, model, fresh_key) + assert served.status_code == 200, served.text + cached: Final = eventually(lambda: cache.get(team_cache_key), lambda value: value is not None, seconds=10) + assert isinstance(cached, bytes), cached + assert json.loads(cached)["team_alias"] == fresh_alias, cached + revoked: Final = _chat(gateway, model, original_key) + assert revoked.status_code == 401, f"{revoked.status_code} {revoked.text}" + + +def test_member_add_and_delete_released_together_leave_no_team_reference(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + newcomer: Final = scenario.user() + team: Final = _create_team(scenario) + with ThreadPoolExecutor(max_workers=2) as pool: + with psycopg.connect(os.environ["DATABASE_URL"]) as holder: + holder.execute(TAKE_TEAM_LOCK_SQL, (team,)) + pending_delete: Final = _post_in_background(pool, gateway, "/team/delete", {"team_ids": [team]}) + pending_add: Final = _post_in_background( + pool, + gateway, + "/team/member_add", + {"team_id": team, "member": {"role": "user", "user_id": newcomer}}, + ) + eventually( + lambda: _waiters_on_lock_held_by(holder.info.backend_pid), + lambda waiting: waiting == 2, + seconds=20, + ) + assert not pending_delete.done() and not pending_add.done() + # leaving the holder block commits its transaction, which releases the advisory lock + deleted: Final = pending_delete.result(timeout=60) + added: Final = pending_add.result(timeout=60) + assert deleted.status_code == 200, deleted.text + assert deleted.json() == {"deleted_teams": [team]} + assert added.status_code in (200, 404), f"{added.status_code} {added.text}" + assert _team_rows(team) == [] + assert _membership_user_ids(team) == [] + assert _user_teams(newcomer) == [] + assert _users_referencing(team) == [] + assert [row["team_id"] for row in _tombstones(team)] == [team] diff --git a/tests/integration/management/test_team_delete_member_cache_eviction.py b/tests/integration/management/test_team_delete_member_cache_eviction.py new file mode 100644 index 00000000000..0b1daa1f527 --- /dev/null +++ b/tests/integration/management/test_team_delete_member_cache_eviction.py @@ -0,0 +1,446 @@ +""" +`/team/delete` cache eviction across both proxies: member user objects, the team object and the +team's keys must stop being served by every worker once the team rows are gone. + +Auth caches the user object under the Redis key ``, the team under `team_id:` +and the key under its sha256; `enable_redis_auth_cache` is on, so Redis is the observable and +the pubsub channel carries the in-memory eviction to the peer proxy. +""" + +import asyncio +import json +import os +import uuid +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from hashlib import sha256 +from typing import Final + +import anthropic +import httpx +import openai +import pytest +from pydantic import JsonValue +from redis import Redis + +from tests.integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + delete_key_if_present, + eventually, + string_value, +) +from tests.integration._support.database import read_rows, write_rows +from tests.integration._support.wire import Reply, Request, wire_server + +_USAGE: Final = {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15} +_CACHE_KEY_HEADER: Final = "x-litellm-cache-key" + + +def _redis() -> Redis: + return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) + + +def _cached_user(cache: Redis, user_id: str) -> dict[str, JsonValue] | None: + raw: Final = cache.get(user_id) + if raw is None: + return None + assert isinstance(raw, bytes), raw + return JSON_OBJECT.validate_json(raw) + + +def _warmed_user(cache: Redis, user_id: str) -> dict[str, JsonValue]: + """The cached user once its Redis SET has landed: auth writes memory at once but sends the Redis + SET on the request's pipeline, so the entry can trail the response that warmed it.""" + cached: Final = eventually(lambda: _cached_user(cache, user_id), lambda value: value is not None, seconds=10) + assert cached is not None + return cached + + +def _delete_team_if_present(gateway: Gateway, team_id: str) -> None: + if read_rows('SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)): + gateway.post("/team/delete", {"team_ids": [team_id]}) + + +def _team(gateway: Gateway, scenario: Scenario) -> str: + """A team the test deletes itself; cleanup removes it only if the test failed before that delete.""" + created: Final = gateway.post("/team/new", {"team_alias": f"integration-{uuid.uuid4().hex}"}) + team_id: Final = string_value(created["team_id"]) + scenario.cleanups.callback(_delete_team_if_present, gateway, team_id) + return team_id + + +def _team_key(gateway: Gateway, scenario: Scenario, team_id: str, model: str) -> str: + """A key `/team/delete` removes; cleanup deletes it only if the team delete never ran.""" + created: Final = gateway.post("/key/generate", {"team_id": team_id, "models": [model]}) + token: Final = string_value(created["key"]) + scenario.cleanups.callback(delete_key_if_present, gateway, token) + return token + + +def _delete_team(gateway: Gateway, team_id: str) -> None: + deleted: Final = gateway.post("/team/delete", {"team_ids": [team_id]}) + assert deleted == {"deleted_teams": [team_id]}, deleted + assert read_rows('SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)) == [] + + +def _chat_body(model: str, text: str, stream: bool = False) -> dict[str, JsonValue]: + body: dict[str, JsonValue] = {"model": model, "messages": [{"role": "user", "content": text}]} + if stream: + body["stream"] = True + return body + + +def _chat(proxy: Gateway, model: str, key: str, text: str) -> httpx.Response: + return proxy.request("POST", "/v1/chat/completions", _chat_body(model, text), key=key) + + +def _team_info(proxy: Gateway, team_id: str) -> httpx.Response: + return proxy.request("GET", "/team/info", params={"team_id": team_id}) + + +@pytest.mark.parametrize("roster_case", ("exact", "lower"), ids=("exact-case", "different-case")) +def test_team_delete_evicts_legacy_email_only_member_from_redis(gateway: Gateway, roster_case: str) -> None: + """A roster entry carrying only an email (pre-backfill legacy shape) still names a cached user; the + delete has to resolve it, in whatever case the roster stored it, and drop that user's cache entry.""" + with gateway.scenario() as scenario, _redis() as cache: + model: Final = scenario.model() + email: Final = f"Legacy-{uuid.uuid4().hex[:12]}@Example.com" + user: Final = scenario.user(user_email=email) + key: Final = scenario.key(user_id=user, models=[model]) + team: Final = _team(gateway, scenario) + roster_email: Final = email if roster_case == "exact" else email.lower() + assert (roster_email == email) is (roster_case == "exact"), (email, roster_email) + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET members_with_roles = %s::jsonb WHERE team_id = %s', + (json.dumps([{"role": "user", "user_id": None, "user_email": roster_email}]), team), + ) + write_rows('UPDATE "LiteLLM_UserTable" SET teams = array_append(teams, %s) WHERE user_id = %s', (team, user)) + warm: Final = _chat(gateway, model, key, "warm legacy member " + uuid.uuid4().hex) + assert warm.status_code == 200, warm.text + warmed: Final = _warmed_user(cache, user) + assert warmed["teams"] == [team], warmed + + _delete_team(gateway, team) + + eventually(lambda: _cached_user(cache, user), lambda cached: cached is None, seconds=10) + rows: Final = read_rows('SELECT teams FROM "LiteLLM_UserTable" WHERE user_id = %s', (user,)) + assert rows == [{"teams": []}], rows + + +def test_team_delete_evicts_member_cached_on_peer_and_peer_rehydrates_without_the_team( + gateway: Gateway, peer: Gateway +) -> None: + """The peer's in-memory copy of the member is evicted over pubsub: its next request misses locally + and re-caches the user from the db, whose `teams` no longer holds the deleted team.""" + with gateway.scenario() as scenario, _redis() as cache: + model: Final = scenario.model() + user: Final = scenario.user(user_role="internal_user") + team: Final = _team(gateway, scenario) + gateway.post("/team/member_add", {"team_id": team, "member": {"role": "user", "user_id": user}}) + key: Final = scenario.key(user_id=user, models=[model]) + warm: Final = _chat(peer, model, key, "warm member on peer " + uuid.uuid4().hex) + assert warm.status_code == 200, warm.text + warmed: Final = _warmed_user(cache, user) + assert warmed["teams"] == [team], warmed + + _delete_team(gateway, team) + + eventually(lambda: _cached_user(cache, user), lambda cached: cached is None, seconds=10) + + def rehydrate() -> dict[str, JsonValue] | None: + # A peer worker still holding the stale in-memory copy answers from it and never + # rewrites Redis, so each poll issues a fresh request rather than re-reading Redis alone. + response: Final = _chat(peer, model, key, "rehydrate member on peer " + uuid.uuid4().hex) + assert response.status_code == 200, response.text + return _cached_user(cache, user) + + rehydrated: Final = eventually(rehydrate, lambda cached: cached is not None, seconds=10) + assert rehydrated is not None and rehydrated["teams"] == [], rehydrated + + +def _sse(events: Sequence[object]) -> tuple[bytes, ...]: + return tuple(b"data: " + json.dumps(event).encode() + b"\n\n" for event in events) + (b"data: [DONE]\n\n",) + + +def _chat_reply(stream: bool) -> Reply: + identity: Final = "chatcmpl-" + uuid.uuid4().hex + if not stream: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "team probe"}, "finish_reason": "stop"} + ], + "usage": _USAGE, + } + ).encode() + ) + head: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"} + return Reply( + content_type="text/event-stream", + chunks=_sse( + ( + {**head, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "team "}}]}, + {**head, "choices": [{"index": 0, "delta": {"content": "probe"}}]}, + {**head, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**head, "choices": [], "usage": _USAGE}, + ) + ), + ) + + +def _responses_reply(stream: bool) -> Reply: + identity: Final = uuid.uuid4().hex + completed: Final = { + "id": "resp_" + identity, + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "type": "message", + "id": "msg_" + identity, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "team probe", "annotations": []}], + } + ], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 11, "output_tokens": 4, "total_tokens": 15}, + } + if not stream: + return Reply(body=json.dumps(completed).encode()) + events: Final = ( + {"type": "response.created", "response": {**completed, "status": "in_progress", "output": [], "usage": None}}, + { + "type": "response.output_text.delta", + "item_id": "msg_" + identity, + "output_index": 0, + "content_index": 0, + "delta": "team probe", + }, + {"type": "response.completed", "response": completed}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events), + ) + + +def _upstream(request: Request) -> Reply: + stream: Final = json.loads(request.body).get("stream") is True + if request.target.endswith("/responses"): + return _responses_reply(stream) + return _chat_reply(stream) + + +def _v1(proxy: Gateway) -> str: + return str(proxy.client.base_url).rstrip("/") + "/v1" + + +def _sdk_status(error: openai.APIStatusError | anthropic.APIStatusError) -> int | str: + if isinstance(error, (openai.AuthenticationError, anthropic.AuthenticationError)): + return error.status_code + return f"{type(error).__name__}:{error.status_code}" + + +def _httpx_chat(proxy: Gateway, model: str, key: str, stream: bool, text: str) -> int | str: + return proxy.request("POST", "/v1/chat/completions", _chat_body(model, text, stream), key=key).status_code + + +def _httpx_messages(proxy: Gateway, model: str, key: str, stream: bool, text: str) -> int | str: + body: Final = {"model": model, "max_tokens": 64, "messages": [{"role": "user", "content": text}], "stream": stream} + return proxy.request("POST", "/v1/messages", body, key=key).status_code + + +def _httpx_responses(proxy: Gateway, model: str, key: str, stream: bool, text: str) -> int | str: + return proxy.request( + "POST", "/v1/responses", {"model": model, "input": text, "stream": stream}, key=key + ).status_code + + +def _openai_sync(proxy: Gateway, model: str, key: str, stream: bool, text: str) -> int | str: + with openai.OpenAI( + api_key=key, base_url=_v1(proxy), max_retries=0, http_client=httpx.Client(timeout=15, trust_env=False) + ) as client: + try: + if stream: + for _ in client.chat.completions.create( + model=model, messages=[{"role": "user", "content": text}], stream=True + ): + pass + else: + client.chat.completions.create(model=model, messages=[{"role": "user", "content": text}]) + except openai.APIStatusError as error: + return _sdk_status(error) + return 200 + + +def _openai_async(proxy: Gateway, model: str, key: str, stream: bool, text: str) -> int | str: + async def call() -> int | str: + async with openai.AsyncOpenAI( + api_key=key, base_url=_v1(proxy), max_retries=0, http_client=httpx.AsyncClient(timeout=15, trust_env=False) + ) as client: + try: + if stream: + async for _ in await client.chat.completions.create( + model=model, messages=[{"role": "user", "content": text}], stream=True + ): + pass + else: + await client.chat.completions.create(model=model, messages=[{"role": "user", "content": text}]) + except openai.APIStatusError as error: + return _sdk_status(error) + return 200 + + return asyncio.run(call()) + + +def _anthropic_sync(proxy: Gateway, model: str, key: str, stream: bool, text: str) -> int | str: + with anthropic.Anthropic( + api_key=key, + base_url=str(proxy.client.base_url), + max_retries=0, + http_client=httpx.Client(timeout=15, trust_env=False), + ) as client: + try: + if stream: + for _ in client.messages.create( + model=model, max_tokens=64, messages=[{"role": "user", "content": text}], stream=True + ): + pass + else: + client.messages.create(model=model, max_tokens=64, messages=[{"role": "user", "content": text}]) + except anthropic.APIStatusError as error: + return _sdk_status(error) + return 200 + + +@dataclass(frozen=True, slots=True) +class _Client: + name: str + call: Callable[[Gateway, str, str, bool, str], int | str] + stream: bool + + +_CLIENTS: Final = ( + _Client("httpx-chat", _httpx_chat, False), + _Client("httpx-chat-stream", _httpx_chat, True), + _Client("httpx-messages", _httpx_messages, False), + _Client("httpx-messages-stream", _httpx_messages, True), + _Client("httpx-responses", _httpx_responses, False), + _Client("httpx-responses-stream", _httpx_responses, True), + _Client("openai-sync", _openai_sync, False), + _Client("openai-sync-stream", _openai_sync, True), + _Client("openai-async", _openai_async, False), + _Client("openai-async-stream", _openai_async, True), + _Client("anthropic-sync", _anthropic_sync, False), + _Client("anthropic-sync-stream", _anthropic_sync, True), +) + + +def _observe(proxies: Mapping[str, Gateway], model: str, key: str) -> dict[str, int | str]: + """One cell per proxy and client; unique text per cell keeps the response cache out of the picture.""" + return { + f"{proxy_name}/{client.name}": client.call( + proxy, model, key, client.stream, f"{client.name} {uuid.uuid4().hex}" + ) + for proxy_name, proxy in proxies.items() + for client in _CLIENTS + } + + +def _off(observed: Mapping[str, int | str], expected: int) -> dict[str, int | str]: + return {cell: status for cell, status in observed.items() if status != expected} + + +def test_team_delete_refuses_the_team_key_for_every_client_on_both_proxies(gateway: Gateway, peer: Gateway) -> None: + """Every surface a deleted team's key can reach, on the primary and on the peer, answers 401 + once the team is gone; every cell is checked and every failing cell is reported at once.""" + proxies: Final = {"primary": gateway, "peer": peer} + with wire_server(_upstream) as upstream, gateway.scenario() as scenario: + model: Final = scenario.model(api_base=upstream.url + "/v1") + team: Final = _team(gateway, scenario) + key: Final = _team_key(gateway, scenario, team, model) + before: Final = _observe(proxies, model, key) + assert _off(before, 200) == {}, _off(before, 200) + + _delete_team(gateway, team) + + eventually( + lambda: _httpx_chat(peer, model, key, False, "deleted team key on peer"), + lambda status: status == 401, + seconds=10, + ) + after: Final = _observe(proxies, model, key) + assert _off(after, 401) == {}, _off(after, 401) + + +def test_team_delete_rejects_the_deleted_key_before_the_response_cache(gateway: Gateway) -> None: + """A request the response cache already answers for this key is refused at auth after the delete: + 401, and the upstream never sees it, so the cache-hit path cannot outlive the key.""" + with wire_server(_upstream) as upstream, gateway.scenario() as scenario: + model: Final = scenario.model(api_base=upstream.url + "/v1") + team: Final = _team(gateway, scenario) + key: Final = _team_key(gateway, scenario, team, model) + marker: Final = "cache twin " + uuid.uuid4().hex + body: Final = _chat_body(model, marker) + first: Final = gateway.request("POST", "/v1/chat/completions", body, key=key) + assert first.status_code == 200, first.text + assert first.headers.get(_CACHE_KEY_HEADER) is None, dict(first.headers) + second: Final = gateway.request("POST", "/v1/chat/completions", body, key=key) + assert second.status_code == 200, second.text + assert second.headers.get(_CACHE_KEY_HEADER), dict(second.headers) + assert second.json()["id"] == first.json()["id"], (first.text, second.text) + received: Final = upstream.drain() + assert len(received) == 1 and marker.encode() in received[0].body, received + + _delete_team(gateway, team) + + third: Final = gateway.request("POST", "/v1/chat/completions", body, key=key) + assert third.status_code == 401, third.text + assert "token_not_found_in_db" in third.text, third.text + assert upstream.drain() == (), "upstream saw a request for the deleted key" + + +def test_team_delete_evicts_team_object_and_key_on_both_proxies(gateway: Gateway, peer: Gateway) -> None: + """Team object and key warm on both proxies before the delete: `/team/info` is 404 and the key is + 401 on both afterwards, and neither the team nor the key entry is left in Redis.""" + with gateway.scenario() as scenario, _redis() as cache: + model: Final = scenario.model() + team: Final = _team(gateway, scenario) + key: Final = _team_key(gateway, scenario, team, model) + hashed: Final = sha256(key.encode()).hexdigest() + for proxy in (gateway, peer): + info: httpx.Response = _team_info(proxy, team) + assert info.status_code == 200 and info.json()["team_id"] == team, info.text + warm: httpx.Response = _chat(proxy, model, key, "warm team key " + uuid.uuid4().hex) + assert warm.status_code == 200, warm.text + # Both SETs ride the warming request's Redis pipeline and can land after its response. + eventually(lambda: cache.exists(f"team_id:{team}"), lambda present: present == 1, seconds=10) + eventually(lambda: cache.exists(hashed), lambda present: present == 1, seconds=10) + + _delete_team(gateway, team) + + eventually(lambda: _team_info(peer, team).status_code, lambda status: status == 404, seconds=10) + eventually( + lambda: _chat(peer, model, key, "deleted team key on peer").status_code, + lambda status: status == 401, + seconds=10, + ) + for proxy in (gateway, peer): + gone: httpx.Response = _team_info(proxy, team) + assert gone.status_code == 404 and "Team not found" in gone.text, gone.text + refused: httpx.Response = _chat(proxy, model, key, "deleted team key " + uuid.uuid4().hex) + assert refused.status_code == 401 and "token_not_found_in_db" in refused.text, refused.text + assert cache.exists(f"team_id:{team}") == 0, cache.keys(f"*{team}*") + assert cache.exists(hashed) == 0, cache.keys(f"*{hashed}*") diff --git a/tests/integration/management/test_team_delete_prometheus.py b/tests/integration/management/test_team_delete_prometheus.py new file mode 100644 index 00000000000..c5e383131f2 --- /dev/null +++ b/tests/integration/management/test_team_delete_prometheus.py @@ -0,0 +1,122 @@ +"""H7: the Prometheus team members gauge follows ``/team/member_add`` and ``/team/delete``. + +An owned single-worker proxy registers the ``prometheus`` callback, so ``GET /metrics/`` serves the +in-process registry (one worker, so no ``PROMETHEUS_MULTIPROC_DIR``). A team with an alias takes three +users in one bulk ``/team/member_add``; the ``litellm_team_members_metric`` series carrying that team's +id then reads 3.0. ``/team/delete`` re-emits the gauge with an empty roster instead of dropping the +series, so the same series afterwards reads 0.0. + +``disable_auto_add_proxy_admin_to_teams`` is on for the owned proxy: a master-key ``/team/new`` +otherwise seeds the roster with ``default_user_id`` and the gauge would read 4.0 after three adds. +""" + +from __future__ import annotations + +import os +import uuid +from pathlib import Path +from typing import Final + +import pytest +import yaml + +from tests.integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy_process + +METRIC: Final = "litellm_team_members_metric" +METRICS_ROUTE: Final = "/metrics/" +MEMBERS: Final = 3 +TEAM_SQL: Final = 'SELECT team_id, members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s' + + +def _prometheus_config(tmp_path: Path) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["litellm_settings"]["callbacks"] = ["prometheus"] + config["general_settings"]["disable_auto_add_proxy_admin_to_teams"] = True + path: Final = tmp_path / "prometheus.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _labels(text: str) -> dict[str, str]: + """``team="a",team_alias="b"`` to ``{"team": "a", "team_alias": "b"}``; ids and aliases carry no commas or quotes.""" + return {name: value.strip('"') for name, _, value in (pair.partition("=") for pair in text.split(","))} + + +def _team_members_series(scrape: str, team_id: str) -> tuple[dict[str, str], float] | None: + """The one ``litellm_team_members_metric`` sample whose ``team`` label is ``team_id``, as (labels, value).""" + samples: Final = tuple( + (labels, float(value)) + for line in scrape.splitlines() + if line.startswith(METRIC + "{") + for label_text, _, value in (line[len(METRIC) + 1 :].partition("} "),) + for labels in (_labels(label_text),) + if labels.get("team") == team_id + ) + assert len(samples) <= 1, f"{METRIC} exported more than one series for team {team_id}: {samples}" + return samples[0] if samples else None + + +def _scrape(candidate: Gateway) -> str: + response: Final = candidate.request("GET", METRICS_ROUTE) + assert response.status_code == 200, f"GET {METRICS_ROUTE}: {response.status_code} {response.text}" + return response.text + + +def _user(candidate: Gateway, scenario: Scenario) -> str: + """An internal user created through ``candidate``; its removal is registered on the shared rig.""" + user_id: Final = f"integration-h7-{uuid.uuid4().hex}" + candidate.post("/user/new", {"user_id": user_id, "auto_create_key": False, "user_role": "internal_user"}) + scenario.cleanups.callback(scenario.delete_user, user_id) + return user_id + + +def _delete_team_if_present(candidate: Gateway, team_id: str) -> None: + if read_rows(TEAM_SQL, (team_id,)): + candidate.post("/team/delete", {"team_ids": [team_id]}) + assert read_rows(TEAM_SQL, (team_id,)) == [] + + +@pytest.mark.timeout(240) # owned proxy boot (prisma db push + readiness) takes 20-40 s +def test_team_members_gauge_reads_roster_size_then_zero_after_delete(gateway: Gateway, tmp_path: Path) -> None: + with ( + gateway.scenario() as scenario, + owned_proxy_process( + gateway, + tmp_path, + {"DATABASE_URL": os.environ["DATABASE_URL"]}, + config=_prometheus_config(tmp_path), + remove_environment=("DATABASE_URL_READ_REPLICA",), + ) as owned, + ): + candidate: Final = owned.gateway + alias: Final = f"integration-h7-{uuid.uuid4().hex}" + team_id: Final = string_value(candidate.post("/team/new", {"team_alias": alias})["team_id"]) + scenario.cleanups.callback(_delete_team_if_present, gateway, team_id) + users: Final = tuple(_user(candidate, scenario) for _ in range(MEMBERS)) + candidate.post( + "/team/member_add", + {"team_id": team_id, "member": [{"role": "user", "user_id": user_id} for user_id in users]}, + ) + rows: Final = read_rows(TEAM_SQL, (team_id,)) + assert len(rows) == 1, rows + roster: Final = rows[0]["members_with_roles"] + assert isinstance(roster, list), roster + assert sorted(string_value(object_value(member)["user_id"]) for member in roster) == sorted(users), roster + + before: Final = eventually( + lambda: _team_members_series(_scrape(candidate), team_id), + lambda sample: sample is not None, + seconds=30, + ) + assert before == ({"team": team_id, "team_alias": alias}, 3.0), before + + assert candidate.post("/team/delete", {"team_ids": [team_id]}) == {"deleted_teams": [team_id]} + assert read_rows(TEAM_SQL, (team_id,)) == [] + after: Final = eventually( + lambda: _team_members_series(_scrape(candidate), team_id), + lambda sample: sample is not None and sample[1] == 0.0, + seconds=30, + ) + assert after == ({"team": team_id, "team_alias": alias}, 0.0), after diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index a53894fcd1b..f2ce01f899e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -8869,10 +8869,6 @@ async def test_delete_team_persists_deleted_teams( "litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin", ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.team_endpoints._team_member_delete", - AsyncMock(return_value=(team1, (), ())), - ) data = DeleteTeamRequest(team_ids=["team-1"]) @@ -9015,6 +9011,113 @@ async def test_delete_team_sweeps_references_outside_members_with_roles( assert cache_state_when_rows_deleted["doomed_still_cached"] is True +def test_delete_team_request_collapses_repeated_ids_in_order(): + """`[T, T, U]` deletes T once and U once: one tombstone, one audit row and one eviction per team.""" + from litellm.proxy._types import DeleteTeamRequest + + assert DeleteTeamRequest(team_ids=["team-a", "team-b", "team-a", "team-b", "team-c"]).team_ids == [ + "team-a", + "team-b", + "team-c", + ] + + +@pytest.mark.asyncio +async def test_delete_team_evicts_member_caches_with_one_transaction( + monkeypatch, + disable_audit_logging_for_mocked_team, +): + """ + Regression pin for LIT-8533: `delete_team` used to fan out one + `_team_member_delete` per roster entry via `asyncio.gather`, and each opened + its own `prisma_client.tx()` and queued on the team's advisory lock, so a + team larger than the Prisma pool exhausted it and the late transactions died + on P2028. Every member-side db effect is already covered by the key delete + and the locked sweep, so the only work left is evicting each member's cache + entries, which needs no transaction at all. + """ + from litellm.proxy._types import DeleteTeamRequest + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + member_user_ids = tuple(f"member-{i}" for i in range(3)) + team = LiteLLM_TeamTable( + team_id="team-doomed", + team_alias="doomed-team", + members_with_roles=[Member(user_id=user_id, role="user") for user_id in member_user_ids] + + [ + Member(user_id=None, user_email="invitee@example.com", role="user"), + Member(user_id=None, user_email="Second.Invitee@Example.com", role="user"), + ], + metadata={}, + model_max_budget={}, + model_spend={}, + ) + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=team) + mock_prisma_client.delete_data = AsyncMock(return_value={"deleted_keys": 0}) + mock_prisma_client.db.litellm_deletedteamtable.create_many = AsyncMock() + mock_prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.execute_raw = AsyncMock() + mock_prisma_client.db.litellm_teammembership.delete_many = AsyncMock() + mock_prisma_client.db.litellm_usertable.find_many = AsyncMock( + return_value=[ + LiteLLM_UserTable(user_id="invited-user", user_email="invitee@example.com"), + LiteLLM_UserTable(user_id="second-invited-user", user_email="second.invitee@example.com"), + ] + ) + + mock_tx = AsyncMock() + mock_tx.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_tx_cm = MagicMock() + mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx) + mock_tx_cm.__aexit__ = AsyncMock(return_value=False) + mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm) + _wire_team_delete_tx(mock_prisma_client) + + fresh_cache = UserApiKeyCache() + for user_id in member_user_ids: + fresh_cache.set_cache(key=user_id, value=UserAPIKeyAuth(user_id=user_id)) + fresh_cache.set_cache(key="invited-user", value=UserAPIKeyAuth(user_id="invited-user")) + fresh_cache.set_cache(key="second-invited-user", value=UserAPIKeyAuth(user_id="second-invited-user")) + fresh_cache.set_cache(key="bystander-user", value=UserAPIKeyAuth(user_id="bystander-user")) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", fresh_cache) + monkeypatch.setattr("litellm.proxy.proxy_server.create_audit_log_for_update", AsyncMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") + + await delete_team( + data=DeleteTeamRequest(team_ids=["team-doomed"]), + http_request=MagicMock(), + user_api_key_dict=UserAPIKeyAuth( + user_id="admin-user", + api_key="sk-admin", + user_role=LitellmUserRoles.PROXY_ADMIN.value, + ), + litellm_changed_by="admin-user", + ) + + assert mock_prisma_client.tx.call_count == 1, ( + f"delete_team must run a single locked transaction for the whole delete, not one per member; " + f"prisma_client.tx() was entered {mock_prisma_client.tx.call_count} times for " + f"{len(member_user_ids)} members" + ) + for user_id in member_user_ids: + assert fresh_cache.get_cache(key=user_id) is None, ( + f"member {user_id}'s cached user object survived the team delete" + ) + for user_id in ("invited-user", "second-invited-user"): + assert fresh_cache.get_cache(key=user_id) is None, ( + f"the email-only roster entry resolving to {user_id} must have its cached user object evicted too" + ) + assert fresh_cache.get_cache(key="bystander-user") is not None + assert mock_prisma_client.db.litellm_usertable.find_many.await_count == 1, ( + "email-only roster entries must resolve in one lookup, not one query per email" + ) + + @pytest.mark.asyncio async def test_delete_team_evicts_the_auth_cache_of_the_keys_it_deletes( monkeypatch, @@ -14146,12 +14249,6 @@ async def test_delete_team_emits_only_the_deleted_audit_event(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") - removals = [(team, members, members[1:]), (team, members[1:], ())] - monkeypatch.setattr( - "litellm.proxy.management_endpoints.team_endpoints._team_member_delete", - AsyncMock(side_effect=lambda **_kwargs: removals.pop(0)), - ) - await delete_team( data=DeleteTeamRequest(team_ids=["team-gone"]), http_request=MagicMock(), diff --git a/tests/unit/repositories/test_repositories.py b/tests/unit/repositories/test_repositories.py index e185d95ffb8..bd0f194b326 100644 --- a/tests/unit/repositories/test_repositories.py +++ b/tests/unit/repositories/test_repositories.py @@ -18,6 +18,7 @@ from litellm.models.credentials import CredentialItem from litellm.models.team import LiteLLM_TeamTable from litellm.repositories.base_repository import BaseRepository from litellm.repositories.budget_repository import BudgetRepository +from litellm.repositories.chunked_in import IN_LIST_CHUNK_SIZE from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.credentials_repository import CredentialsRepository from litellm.repositories.model_repository import ModelRepository @@ -891,6 +892,32 @@ class TestUserRepository: user = await repo.find_by_email("test@example.com") assert user is not None + @pytest.mark.asyncio + async def test_find_by_emails_is_one_case_insensitive_query(self, repo): + repo._prisma_client.db.litellm_usertable.find_many = AsyncMock(return_value=[]) + await repo.find_by_emails(["B@Example.com", "a@example.com", "B@Example.com"]) + repo._prisma_client.db.litellm_usertable.find_many.assert_awaited_once() + where = repo._prisma_client.db.litellm_usertable.find_many.await_args.kwargs["where"] + assert where["user_email"] == {"in": ["B@Example.com", "a@example.com"], "mode": "insensitive"} + + @pytest.mark.asyncio + async def test_find_by_emails_slices_the_list_into_bounded_statements(self, repo): + repo._prisma_client.db.litellm_usertable.find_many = AsyncMock(return_value=[]) + emails = [f"user{index}@example.com" for index in range(IN_LIST_CHUNK_SIZE + 1)] + await repo.find_by_emails(emails) + assert repo._prisma_client.db.litellm_usertable.find_many.await_count == 2 + sizes = [ + len(call.kwargs["where"]["user_email"]["in"]) + for call in repo._prisma_client.db.litellm_usertable.find_many.await_args_list + ] + assert sizes == [IN_LIST_CHUNK_SIZE, 1] + + @pytest.mark.asyncio + async def test_find_by_emails_skips_the_query_for_no_emails(self, repo): + repo._prisma_client.db.litellm_usertable.find_many = AsyncMock() + assert await repo.find_by_emails(()) == () + repo._prisma_client.db.litellm_usertable.find_many.assert_not_awaited() + @pytest.mark.asyncio async def test_find_by_sso_id(self, repo): repo._prisma_client.db.litellm_usertable._records["sso-123"] = { From 268eb4d6e62d56bdf4cc08f055b61a5c8f4c4b5e Mon Sep 17 00:00:00 2001 From: yujonglee Date: Wed, 30 Sep 2026 14:12:29 -0700 Subject: [PATCH 5/7] feat(tracing): add OTLP trace ingestion and reads (#43915) --- .github/workflows/test-unit.yml | 4 +- litellm-rust/Cargo.lock | 67 +- litellm-rust/crates/python-bridge/src/lib.rs | 7 +- .../crates/python-bridge/src/routes/traces.rs | 166 ++-- litellm-rust/crates/traces/AGENTS.md | 1 + litellm-rust/crates/traces/Cargo.toml | 4 + litellm-rust/crates/traces/config/reader.xml | 2 +- .../traces/migrations/0001_otel_traces.sql | 2 +- .../traces/migrations/0002_agent_traces.sql | 6 +- .../migrations/0003_agent_traces_mv.sql | 7 +- .../migrations/0006_agent_traces_ttl.sql | 2 +- litellm-rust/crates/traces/src/error.rs | 14 + litellm-rust/crates/traces/src/insert.rs | 138 ++- litellm-rust/crates/traces/src/lib.rs | 37 +- litellm-rust/crates/traces/src/otlp.rs | 221 +++++ litellm-rust/crates/traces/src/schema.rs | 23 +- litellm-rust/crates/traces/tests/admin_sql.rs | 14 + .../crates/traces/tests/migrations.rs | 112 ++- litellm-rust/crates/traces/tests/otlp.rs | 47 + litellm/constants.py | 13 + .../clickhouse/clickhouse_batch_logger.py | 100 ++ litellm/integrations/clickhouse/schema.py | 11 + litellm/integrations/custom_batch_logger.py | 2 +- litellm/proxy/_types.py | 4 + .../proxy/common_utils/http_parsing_utils.py | 22 +- litellm/proxy/proxy_server.py | 28 + litellm/proxy/tracing_endpoints.py | 141 +++ litellm/rust_bridge/_native.pyi | 28 +- litellm/rust_bridge/traces.py | 107 +- litellm/tracing/AGENTS.md | 6 + litellm/tracing/__init__.py | 16 + litellm/tracing/decode.py | 284 ++++++ litellm/tracing/receiver.py | 120 +++ litellm/tracing/store.py | 286 ++++++ litellm/tracing/types.py | 120 +++ .../test_clickhouse_batch_logger.py | 67 ++ .../common_utils/test_http_parsing_utils.py | 29 +- .../proxy/test_tracing_endpoints.py | 180 ++++ .../fixtures/langsmith_deep_agent_export.json | 924 ++++++++++++++++++ tests/test_litellm/tracing/test_decode.py | 313 ++++++ tests/test_litellm/tracing/test_receiver.py | 117 +++ tests/test_litellm/tracing/test_store.py | 311 ++++++ tests/test_litellm_rust/test_traces.py | 51 +- ui/litellm-dashboard/package-lock.json | 82 +- ui/litellm-dashboard/package.json | 2 +- .../src/components/networking.tsx | 15 +- .../TraceView/AgentTracesSection.tsx | 19 +- .../view_logs/TraceView/AgentTracesTable.tsx | 6 +- .../view_logs/TraceView/DetailContent.tsx | 11 +- .../view_logs/TraceView/DetailPane.test.tsx | 2 +- .../view_logs/TraceView/DetailPane.tsx | 17 +- .../view_logs/TraceView/TraceDrawer.tsx | 19 +- .../view_logs/TraceView/traceTypes.ts | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 170 ++++ uv.lock | 12 +- 55 files changed, 4240 insertions(+), 270 deletions(-) create mode 100644 litellm-rust/crates/traces/src/otlp.rs create mode 100644 litellm-rust/crates/traces/tests/otlp.rs create mode 100644 litellm/integrations/clickhouse/clickhouse_batch_logger.py create mode 100644 litellm/integrations/clickhouse/schema.py create mode 100644 litellm/proxy/tracing_endpoints.py create mode 100644 litellm/tracing/AGENTS.md create mode 100644 litellm/tracing/__init__.py create mode 100644 litellm/tracing/decode.py create mode 100644 litellm/tracing/receiver.py create mode 100644 litellm/tracing/store.py create mode 100644 litellm/tracing/types.py create mode 100644 tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py create mode 100644 tests/test_litellm/proxy/test_tracing_endpoints.py create mode 100644 tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json create mode 100644 tests/test_litellm/tracing/test_decode.py create mode 100644 tests/test_litellm/tracing/test_receiver.py create mode 100644 tests/test_litellm/tracing/test_store.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f55e186e3b2..d3604a81e8f 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -79,7 +79,9 @@ jobs: - shard: integrations artifact-name: integrations - test-path: "" + test-path: >- + tests/test_litellm/integrations + tests/test_litellm/tracing unit-flag: integrations workers: 2 reruns: 3 diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 5fdf0266e8c..474f45ac0cd 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1274,6 +1274,18 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" +[[package]] +name = "const-hex" +version = "1.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e59eef12462b0f9b0a3620219be5d639afd79fe39dff0a42c3997061f9298b4" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "proptest", + "serde_core", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -2372,9 +2384,9 @@ dependencies = [ "http-body-util", "hyper 1.10.1", "lazy_static", - "opentelemetry", + "opentelemetry 0.32.0", "opentelemetry-semantic-conventions", - "opentelemetry_sdk", + "opentelemetry_sdk 0.32.1", "percent-encoding", "pin-project", "prost", @@ -4356,7 +4368,11 @@ dependencies = [ name = "litellm-traces" version = "0.1.0" dependencies = [ + "base64 0.22.1", + "flate2", "litellm-http", + "opentelemetry-proto", + "prost", "rstest", "serde", "serde_json", @@ -4776,6 +4792,33 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 2.0.19", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" +dependencies = [ + "base64 0.22.1", + "const-hex", + "opentelemetry 0.33.0", + "opentelemetry_sdk 0.33.0", + "prost", + "serde", +] + [[package]] name = "opentelemetry-semantic-conventions" version = "0.32.1" @@ -4791,7 +4834,23 @@ dependencies = [ "futures-channel", "futures-executor", "futures-util", - "opentelemetry", + "opentelemetry 0.32.0", + "percent-encoding", + "portable-atomic", + "rand 0.9.5", + "thiserror 2.0.19", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry 0.33.0", "percent-encoding", "portable-atomic", "rand 0.9.5", @@ -7524,7 +7583,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", - "opentelemetry", + "opentelemetry 0.32.0", "tracing", "tracing-core", "tracing-subscriber", diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index f65cb350cfe..0d4df996552 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -44,7 +44,7 @@ mod _native { #[pymodule_export] use crate::routes::token_counter::TokenCounter; #[pymodule_export] - use crate::routes::traces::{trace_encode_rows, trace_ensure_schema, trace_query}; + use crate::routes::traces::{NativeTraceStorage, trace_decode_otlp}; #[cfg(feature = "huggingface")] #[pymodule_export] use crate::tokenizer::HuggingFaceEncoding; @@ -109,9 +109,8 @@ mod tests { "aresponses", "ResponsesWebSocketConnection", "NativeDiagnosticProcessor", - "trace_encode_rows", - "trace_ensure_schema", - "trace_query", + "NativeTraceStorage", + "trace_decode_otlp", "TokenCounter", "Tokenizer", "gil_stats", diff --git a/litellm-rust/crates/python-bridge/src/routes/traces.rs b/litellm-rust/crates/python-bridge/src/routes/traces.rs index fdd2a7eb93d..6a18273ed4c 100644 --- a/litellm-rust/crates/python-bridge/src/routes/traces.rs +++ b/litellm-rust/crates/python-bridge/src/routes/traces.rs @@ -1,19 +1,21 @@ use std::collections::BTreeMap; use litellm_http::ClientVariant; -use litellm_traces::{Connection, Error, Parameter}; +use litellm_traces::{Connection, Error, InsertTable, Parameter}; use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, + exceptions::{PyOverflowError, PyRuntimeError, PyValueError}, prelude::*, }; fn map_error(error: Error) -> PyErr { match error { - Error::InvalidRow | Error::InvalidSchema | Error::EmptySql => { + Error::InvalidRow | Error::InvalidTable | Error::InvalidSchema | Error::EmptySql => { PyValueError::new_err(error.to_string()) } + Error::InsertTooLarge => PyOverflowError::new_err(error.to_string()), Error::InvalidUrl | Error::QueryFailed(_) + | Error::InsertFailed(_) | Error::SchemaFailed(_) | Error::ResponseTooLarge | Error::InvalidResponse @@ -21,63 +23,115 @@ fn map_error(error: Error) -> PyErr { } } -#[pyfunction] -pub fn trace_ensure_schema<'py>( - py: Python<'py>, - url: &str, +#[pyclass] +pub struct NativeTraceStorage { database: String, - user: &str, - password: &str, - trace_retention_days: u32, - spend_log_retention_days: u32, -) -> PyResult> { - let connection = Connection::writer(url, user, password).map_err(map_error)?; - let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; - crate::execution::run_async( - py, - async move { - litellm_traces::ensure_schema( - &client, - &connection, - &database, - trace_retention_days, - spend_log_retention_days, - ) - .await - }, - map_error, - ) + writer: Connection, + reader: Option, +} + +#[pymethods] +impl NativeTraceStorage { + #[new] + fn new(database: String, url: &str, reader_url: Option<&str>) -> PyResult { + litellm_traces::schema_statements(&database, 1, 1).map_err(map_error)?; + Ok(Self { + writer: Connection::writer(url).map_err(map_error)?, + reader: reader_url + .map(|value| Connection::reader(value, &database)) + .transpose() + .map_err(map_error)?, + database, + }) + } + + fn ensure_schema<'py>( + &self, + py: Python<'py>, + trace_retention_days: u32, + spend_log_retention_days: u32, + ) -> PyResult> { + let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; + let connection = self.writer.clone(); + let database = self.database.clone(); + crate::execution::run_async( + py, + async move { + litellm_traces::ensure_schema( + &client, + &connection, + &database, + trace_retention_days, + spend_log_retention_days, + ) + .await + }, + map_error, + ) + } + + fn insert_rows<'py>( + &self, + py: Python<'py>, + table: &str, + #[pyo3(from_py_with = litellm_host_python::from_py_argument)] rows: Vec< + BTreeMap, + >, + ) -> PyResult> { + let table = InsertTable::parse(table).map_err(map_error)?; + let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; + let connection = self.writer.clone(); + let database = self.database.clone(); + crate::execution::run_async( + py, + async move { + litellm_traces::insert_rows(&client, &connection, &database, table, rows).await + }, + map_error, + ) + } + + fn query<'py>( + &self, + py: Python<'py>, + sql: String, + #[pyo3(from_py_with = litellm_host_python::from_py_argument)] parameters: BTreeMap< + String, + Parameter, + >, + ) -> PyResult> { + let connection = self.reader.clone().ok_or_else(|| { + PyRuntimeError::new_err("Trace reads require a separate ClickHouse reader URL") + })?; + let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; + crate::execution::run_async( + py, + async move { litellm_traces::execute_read(&client, &connection, &sql, ¶meters).await }, + map_error, + ) + } } #[pyfunction] -pub fn trace_query<'py>( +pub fn trace_decode_otlp<'py>( py: Python<'py>, - url: &str, - database: &str, - user: &str, - password: &str, - sql: String, - #[pyo3(from_py_with = litellm_host_python::from_py_argument)] parameters: BTreeMap< - String, - Parameter, - >, + body: &[u8], + content_type: Option<&str>, + content_encoding: Option<&str>, + max_decompressed_bytes: usize, ) -> PyResult> { - let connection = Connection::configured(url, database, user, password).map_err(map_error)?; - let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; - crate::execution::run_async( - py, - async move { litellm_traces::execute_read(&client, &connection, &sql, ¶meters).await }, - map_error, - ) -} - -#[pyfunction] -pub fn trace_encode_rows( - py: Python<'_>, - #[pyo3(from_py_with = litellm_host_python::from_py_argument)] rows: Vec< - BTreeMap, - >, -) -> PyResult { - py.detach(|| litellm_traces::encode_rows(rows)) - .map_err(map_error) + let spans = py + .detach(|| { + litellm_traces::decode_otlp( + body, + content_type, + content_encoding, + max_decompressed_bytes, + ) + }) + .map_err(|error| match error { + litellm_traces::DecodeError::TooLarge => PyOverflowError::new_err(error.to_string()), + _ => PyValueError::new_err(error.to_string()), + })?; + litellm_host_python::Pythonized(spans).into_pyobject(py) } diff --git a/litellm-rust/crates/traces/AGENTS.md b/litellm-rust/crates/traces/AGENTS.md index d330429b315..a5e2d4be53a 100644 --- a/litellm-rust/crates/traces/AGENTS.md +++ b/litellm-rust/crates/traces/AGENTS.md @@ -2,5 +2,6 @@ - Keep this crate independent of Python; PyO3 conversion and public Python exceptions belong in `python-bridge` - Keep the SQL migrations here as the only ClickHouse schema definition - Use typed query parameters and a dedicated SELECT-only reader with server-side limits +- Keep `config/reader.xml` grants on the database the schema is created in (CLICKHOUSE_DATABASE, default `litellm`) - Bound insert time and encoded bytes; make retry deduplication behavior explicit for supported ClickHouse versions - Test storage behavior through the crate's public API against ClickHouse diff --git a/litellm-rust/crates/traces/Cargo.toml b/litellm-rust/crates/traces/Cargo.toml index a8091cf9ec8..0aeec4c8276 100644 --- a/litellm-rust/crates/traces/Cargo.toml +++ b/litellm-rust/crates/traces/Cargo.toml @@ -6,6 +6,10 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true +flate2.workspace = true +opentelemetry-proto = { version = "0.33.0", default-features = false, features = ["gen-tonic-messages", "trace", "with-serde"] } +prost = "0.14.4" time = { workspace = true, features = ["formatting"] } litellm-http.workspace = true serde.workspace = true diff --git a/litellm-rust/crates/traces/config/reader.xml b/litellm-rust/crates/traces/config/reader.xml index 73a63035e2e..3ab337a13fc 100644 --- a/litellm-rust/crates/traces/config/reader.xml +++ b/litellm-rust/crates/traces/config/reader.xml @@ -24,7 +24,7 @@ litellm_traces_reader GRANT SELECT ON litellm.otel_traces - GRANT SELECT ON litellm.agent_traces + GRANT SELECT ON litellm.agent_traces_by_key GRANT SELECT ON litellm.spend_logs diff --git a/litellm-rust/crates/traces/migrations/0001_otel_traces.sql b/litellm-rust/crates/traces/migrations/0001_otel_traces.sql index aed869e6ee0..d8e0184b5a3 100644 --- a/litellm-rust/crates/traces/migrations/0001_otel_traces.sql +++ b/litellm-rust/crates/traces/migrations/0001_otel_traces.sql @@ -44,4 +44,4 @@ CREATE TABLE IF NOT EXISTS {database}.otel_traces ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (TeamId, ServiceName, toDateTime(Timestamp), TraceId) -SETTINGS ttl_only_drop_parts = 1 +SETTINGS ttl_only_drop_parts = 1, non_replicated_deduplication_window = 1000 diff --git a/litellm-rust/crates/traces/migrations/0002_agent_traces.sql b/litellm-rust/crates/traces/migrations/0002_agent_traces.sql index d994829f9c0..0c3547872bb 100644 --- a/litellm-rust/crates/traces/migrations/0002_agent_traces.sql +++ b/litellm-rust/crates/traces/migrations/0002_agent_traces.sql @@ -1,6 +1,7 @@ -CREATE TABLE IF NOT EXISTS {database}.agent_traces +CREATE TABLE IF NOT EXISTS {database}.agent_traces_by_key ( TeamId LowCardinality(String), + ApiKeyHash String, TraceId String, StartTs SimpleAggregateFunction(min, DateTime64(9)), EndTs SimpleAggregateFunction(max, DateTime64(9)), @@ -20,4 +21,5 @@ CREATE TABLE IF NOT EXISTS {database}.agent_traces RequestIds SimpleAggregateFunction(groupArrayArray, Array(String)) ) ENGINE = AggregatingMergeTree -ORDER BY (TeamId, TraceId) +ORDER BY (TeamId, ApiKeyHash, TraceId) +SETTINGS non_replicated_deduplication_window = 1000 diff --git a/litellm-rust/crates/traces/migrations/0003_agent_traces_mv.sql b/litellm-rust/crates/traces/migrations/0003_agent_traces_mv.sql index e162d4da569..94dad81f998 100644 --- a/litellm-rust/crates/traces/migrations/0003_agent_traces_mv.sql +++ b/litellm-rust/crates/traces/migrations/0003_agent_traces_mv.sql @@ -1,6 +1,7 @@ -CREATE MATERIALIZED VIEW IF NOT EXISTS {database}.agent_traces_mv TO {database}.agent_traces AS +CREATE MATERIALIZED VIEW IF NOT EXISTS {database}.agent_traces_by_key_mv +TO {database}.agent_traces_by_key AS SELECT - TeamId, TraceId, + TeamId, ApiKeyHash, TraceId, min(Timestamp) AS StartTs, max(Timestamp + toIntervalNanosecond(Duration)) AS EndTs, any(ServiceName) AS ServiceName, @@ -18,4 +19,4 @@ SELECT groupUniqArrayIf(SpanName, ObservationType = 'agent') AS AgentNames, groupArrayIf(LiteLLMRequestId, LiteLLMRequestId != '') AS RequestIds FROM {database}.otel_traces -GROUP BY TeamId, TraceId +GROUP BY TeamId, ApiKeyHash, TraceId diff --git a/litellm-rust/crates/traces/migrations/0006_agent_traces_ttl.sql b/litellm-rust/crates/traces/migrations/0006_agent_traces_ttl.sql index d4c4a113329..8681f0622a4 100644 --- a/litellm-rust/crates/traces/migrations/0006_agent_traces_ttl.sql +++ b/litellm-rust/crates/traces/migrations/0006_agent_traces_ttl.sql @@ -1 +1 @@ -ALTER TABLE {database}.agent_traces MODIFY TTL toDateTime(StartTs) + INTERVAL {trace_retention_days} DAY +ALTER TABLE {database}.agent_traces_by_key MODIFY TTL toDateTime(StartTs) + INTERVAL {trace_retention_days} DAY diff --git a/litellm-rust/crates/traces/src/error.rs b/litellm-rust/crates/traces/src/error.rs index a06eb27397a..125edc35422 100644 --- a/litellm-rust/crates/traces/src/error.rs +++ b/litellm-rust/crates/traces/src/error.rs @@ -2,6 +2,8 @@ pub enum Error { #[error("invalid ClickHouse insert row")] InvalidRow, + #[error("invalid ClickHouse insert table")] + InvalidTable, #[error("invalid ClickHouse HTTP URL")] InvalidUrl, #[error("database must be a nonempty SQL identifier and retention must be positive")] @@ -10,6 +12,10 @@ pub enum Error { EmptySql, #[error("ClickHouse query failed with HTTP status {0}")] QueryFailed(u16), + #[error("ClickHouse insert failed with HTTP status {0}")] + InsertFailed(u16), + #[error("ClickHouse insert exceeds the encoded size limit")] + InsertTooLarge, #[error("ClickHouse schema setup failed with HTTP status {0}")] SchemaFailed(u16), #[error("ClickHouse query exceeded the response size limit")] @@ -19,3 +25,11 @@ pub enum Error { #[error("ClickHouse query transport failed")] Transport, } + +#[derive(Debug, thiserror::Error)] +pub enum DecodeError { + #[error("invalid OTLP trace payload")] + InvalidPayload, + #[error("OTLP trace payload exceeds the decompressed size limit")] + TooLarge, +} diff --git a/litellm-rust/crates/traces/src/insert.rs b/litellm-rust/crates/traces/src/insert.rs index 05b14651ce0..6f2d6acb023 100644 --- a/litellm-rust/crates/traces/src/insert.rs +++ b/litellm-rust/crates/traces/src/insert.rs @@ -1,21 +1,109 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, io::Write, time::Duration}; +use flate2::{Compression, write::GzEncoder}; +use litellm_http::Client; use serde_json::Value; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; -use crate::Error; +use crate::{Connection, Error}; + +const MAX_INSERT_BYTES: usize = 64 * 1024 * 1024; +const INSERT_TIMEOUT: Duration = Duration::from_secs(30); + +pub enum InsertTable { + OtelTraces, + SpendLogs, +} + +impl InsertTable { + pub fn parse(value: &str) -> Result { + match value { + "otel_traces" => Ok(Self::OtelTraces), + "spend_logs" => Ok(Self::SpendLogs), + _ => Err(Error::InvalidTable), + } + } + + fn name(&self) -> &'static str { + match self { + Self::OtelTraces => "otel_traces", + Self::SpendLogs => "spend_logs", + } + } +} + +pub async fn insert_rows( + client: &Client, + connection: &Connection, + database: &str, + table: InsertTable, + rows: Vec>, +) -> Result<(), Error> { + if rows.is_empty() { + return Ok(()); + } + let encoded = encode_rows_with_limit(rows, MAX_INSERT_BYTES)?; + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder + .write_all(encoded.as_bytes()) + .map_err(|_| Error::InvalidRow)?; + let body = encoder.finish().map_err(|_| Error::InvalidRow)?; + let mut url = connection.url().clone(); + url.query_pairs_mut() + .append_pair( + "query", + &format!( + "INSERT INTO `{database}`.{} FORMAT JSONEachRow", + table.name() + ), + ) + .append_pair("async_insert", "1") + .append_pair("async_insert_deduplicate", "1") + .append_pair("wait_for_async_insert", "1") + .append_pair("date_time_input_format", "best_effort"); + let response = client + .post(url) + .timeout(INSERT_TIMEOUT) + .header("Content-Encoding", "gzip") + .body(body) + .send() + .await + .map_err(|_| Error::Transport)?; + if !response.status().is_success() { + return Err(Error::InsertFailed(response.status().as_u16())); + } + Ok(()) +} pub fn encode_rows(rows: Vec>) -> Result { - rows.into_iter() - .map(|row| { - let encoded = row - .into_iter() - .map(|(name, value)| insert_value(&name, value).map(|value| (name, value))) - .collect::, _>>()?; - serde_json::to_string(&encoded).map_err(|_| Error::InvalidRow) - }) - .collect::, _>>() - .map(|rows| rows.join("\n")) + encode_rows_with_limit(rows, usize::MAX) +} + +fn encode_rows_with_limit( + rows: Vec>, + limit: usize, +) -> Result { + let mut body = Vec::new(); + for row in rows { + let encoded = row + .into_iter() + .map(|(name, value)| insert_value(&name, value).map(|value| (name, value))) + .collect::, _>>()?; + let record = serde_json::to_vec(&encoded).map_err(|_| Error::InvalidRow)?; + let size = body + .len() + .checked_add(record.len()) + .and_then(|size| size.checked_add(usize::from(!body.is_empty()))) + .ok_or(Error::InsertTooLarge)?; + if size > limit { + return Err(Error::InsertTooLarge); + } + if !body.is_empty() { + body.push(b'\n'); + } + body.extend_from_slice(&record); + } + String::from_utf8(body).map_err(|_| Error::InvalidRow) } fn insert_value(name: &str, value: Value) -> Result { @@ -35,3 +123,29 @@ fn insert_value(name: &str, value: Value) -> Result { .map(Value::String) .map_err(|_| Error::InvalidRow) } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use rstest::rstest; + use serde_json::json; + + use super::encode_rows_with_limit; + use crate::Error; + + #[rstest] + fn encoded_limit_counts_utf8_bytes_across_rows() { + let rows = vec![ + BTreeMap::from([("Input".to_owned(), json!("雪"))]), + BTreeMap::from([("Input".to_owned(), json!("雪"))]), + ]; + let encoded = encode_rows_with_limit(rows.clone(), usize::MAX).expect("valid rows"); + + assert!(encode_rows_with_limit(rows.clone(), encoded.len()).is_ok()); + assert!(matches!( + encode_rows_with_limit(rows, encoded.len() - 1), + Err(Error::InsertTooLarge) + )); + } +} diff --git a/litellm-rust/crates/traces/src/lib.rs b/litellm-rust/crates/traces/src/lib.rs index 1f56bb5b3a6..279afb20e9b 100644 --- a/litellm-rust/crates/traces/src/lib.rs +++ b/litellm-rust/crates/traces/src/lib.rs @@ -1,10 +1,12 @@ mod error; mod insert; +mod otlp; mod schema; mod sql; -pub use error::Error; -pub use insert::encode_rows; +pub use error::{DecodeError, Error}; +pub use insert::{InsertTable, encode_rows, insert_rows}; +pub use otlp::{DecodedSpan, decode_otlp}; pub use schema::{ensure_schema, schema_statements}; pub use sql::{Parameter, execute_read}; use url::Url; @@ -53,17 +55,32 @@ impl Connection { Ok(connection) } - pub fn writer(url: &str, user: &str, password: &str) -> Result { + pub fn writer(url: &str) -> Result { let mut connection = Self::parse(url)?; + let pairs: Vec<_> = connection + .url + .query_pairs() + .filter(|(key, _)| !matches!(key.as_ref(), "database" | "readonly" | "query")) + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); + connection.url.query_pairs_mut().clear().extend_pairs(pairs); + Ok(connection) + } + + pub fn reader(url: &str, database: &str) -> Result { + let mut connection = Self::parse(url)?; + let pairs: Vec<_> = connection + .url + .query_pairs() + .filter(|(key, _)| key != "database") + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); connection .url - .set_username(user) - .map_err(|_| Error::InvalidUrl)?; - connection - .url - .set_password(Some(password)) - .map_err(|_| Error::InvalidUrl)?; - connection.url.set_query(None); + .query_pairs_mut() + .clear() + .extend_pairs(pairs) + .append_pair("database", database); Ok(connection) } diff --git a/litellm-rust/crates/traces/src/otlp.rs b/litellm-rust/crates/traces/src/otlp.rs new file mode 100644 index 00000000000..f162256ef1f --- /dev/null +++ b/litellm-rust/crates/traces/src/otlp.rs @@ -0,0 +1,221 @@ +use std::{collections::BTreeMap, io::Read}; + +use base64::Engine; +use flate2::read::GzDecoder; +use opentelemetry_proto::tonic::{ + collector::trace::v1::ExportTraceServiceRequest, + common::v1::{AnyValue, KeyValue, any_value::Value as AttributeValue}, + trace::v1::{Span, span::SpanKind, status::StatusCode}, +}; +use prost::Message; +use serde::Serialize; +use serde_json::Value; + +use crate::DecodeError; + +#[derive(Serialize)] +pub struct DecodedEvent { + pub name: String, + pub attributes: BTreeMap, +} + +#[derive(Serialize)] +pub struct DecodedSpan { + pub trace_id: String, + pub span_id: String, + pub parent_span_id: String, + pub trace_state: String, + pub name: String, + pub kind: String, + pub resource_attributes: BTreeMap, + pub scope_name: String, + pub scope_version: String, + pub attributes: BTreeMap, + pub start_ns: u64, + pub end_ns: u64, + pub status_code: String, + pub status_message: String, + pub events: Vec, +} + +pub fn decode_otlp( + body: &[u8], + content_type: Option<&str>, + content_encoding: Option<&str>, + max_decompressed_bytes: usize, +) -> Result, DecodeError> { + let payload = if content_encoding == Some("gzip") || body.starts_with(&[0x1f, 0x8b]) { + let limit = u64::try_from(max_decompressed_bytes).map_err(|_| DecodeError::TooLarge)?; + let mut decoded = Vec::new(); + GzDecoder::new(body) + .take(limit + 1) + .read_to_end(&mut decoded) + .map_err(|_| DecodeError::InvalidPayload)?; + decoded + } else { + body.to_vec() + }; + if payload.len() > max_decompressed_bytes { + return Err(DecodeError::TooLarge); + } + let request = if content_type.is_some_and(|value| value.contains("json")) { + let value: Value = + serde_json::from_slice(&payload).map_err(|_| DecodeError::InvalidPayload)?; + serde_json::from_value(normalize_json_ids(value)?) + .map_err(|_| DecodeError::InvalidPayload)? + } else { + ExportTraceServiceRequest::decode(payload.as_slice()) + .map_err(|_| DecodeError::InvalidPayload)? + }; + Ok(request + .resource_spans + .into_iter() + .flat_map(|resource_spans| { + let resource_attributes = attributes( + resource_spans + .resource + .map(|resource| resource.attributes) + .unwrap_or_default(), + ); + resource_spans + .scope_spans + .into_iter() + .flat_map(move |scope_spans| { + let scope = scope_spans.scope.unwrap_or_default(); + let resource_attributes = resource_attributes.clone(); + scope_spans.spans.into_iter().map(move |span| { + decoded_span(span, &resource_attributes, &scope.name, &scope.version) + }) + }) + }) + .collect()) +} + +fn normalize_json_ids(value: Value) -> Result { + match value { + Value::Object(fields) => fields + .into_iter() + .map(|(name, value)| { + let normalized = if matches!(name.as_str(), "traceId" | "spanId" | "parentSpanId") { + let encoded = value.as_str().ok_or(DecodeError::InvalidPayload)?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|_| DecodeError::InvalidPayload)?; + Value::String(hex_bytes(&bytes)) + } else if name == "kind" && value.is_string() { + let kind = SpanKind::from_str_name(value.as_str().unwrap_or_default()) + .ok_or(DecodeError::InvalidPayload)?; + Value::from(kind as i32) + } else if name == "code" && value.is_string() { + let code = StatusCode::from_str_name(value.as_str().unwrap_or_default()) + .ok_or(DecodeError::InvalidPayload)?; + Value::from(code as i32) + } else { + normalize_json_ids(value)? + }; + Ok((name, normalized)) + }) + .collect::, _>>() + .map(Value::Object), + Value::Array(values) => values + .into_iter() + .map(normalize_json_ids) + .collect::, _>>() + .map(Value::Array), + value => Ok(value), + } +} + +fn hex_bytes(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn decoded_span( + span: Span, + resource_attributes: &BTreeMap, + scope_name: &str, + scope_version: &str, +) -> DecodedSpan { + let status = span.status.unwrap_or_default(); + DecodedSpan { + trace_id: hex_bytes(&span.trace_id), + span_id: hex_bytes(&span.span_id), + parent_span_id: hex_bytes(&span.parent_span_id), + trace_state: span.trace_state, + name: span.name, + kind: SpanKind::try_from(span.kind) + .unwrap_or(SpanKind::Unspecified) + .as_str_name() + .to_owned(), + resource_attributes: resource_attributes.clone(), + scope_name: scope_name.to_owned(), + scope_version: scope_version.to_owned(), + attributes: attributes(span.attributes), + start_ns: span.start_time_unix_nano, + end_ns: span.end_time_unix_nano, + status_code: StatusCode::try_from(status.code) + .unwrap_or(StatusCode::Unset) + .as_str_name() + .to_owned(), + status_message: status.message, + events: span + .events + .into_iter() + .map(|event| DecodedEvent { + name: event.name, + attributes: attributes(event.attributes), + }) + .collect(), + } +} + +fn attributes(values: Vec) -> BTreeMap { + values + .into_iter() + .map(|entry| { + ( + entry.key, + entry.value.as_ref().map(attribute_text).unwrap_or_default(), + ) + }) + .collect() +} + +fn attribute_text(value: &AnyValue) -> String { + match value.value.as_ref() { + Some(AttributeValue::StringValue(value)) => value.clone(), + Some(AttributeValue::BoolValue(value)) => value.to_string(), + Some(AttributeValue::IntValue(value)) => value.to_string(), + Some(AttributeValue::DoubleValue(value)) => { + serde_json::to_string(value).unwrap_or_default() + } + Some(AttributeValue::BytesValue(value)) => String::from_utf8_lossy(value).into_owned(), + Some(AttributeValue::ArrayValue(value)) => format!( + "[{}]", + value + .values + .iter() + .map(|value| serde_json::to_string(&attribute_text(value)).unwrap_or_default()) + .collect::>() + .join(", ") + ), + Some(AttributeValue::KvlistValue(value)) => format!( + "{{{}}}", + value + .values + .iter() + .map(|entry| format!( + "{}: {}", + serde_json::to_string(&entry.key).unwrap_or_default(), + serde_json::to_string( + &entry.value.as_ref().map(attribute_text).unwrap_or_default() + ) + .unwrap_or_default() + )) + .collect::>() + .join(", ") + ), + Some(AttributeValue::StringValueStrindex(value)) => value.to_string(), + None => String::new(), + } +} diff --git a/litellm-rust/crates/traces/src/schema.rs b/litellm-rust/crates/traces/src/schema.rs index 5258c86ca78..5a154eb87c3 100644 --- a/litellm-rust/crates/traces/src/schema.rs +++ b/litellm-rust/crates/traces/src/schema.rs @@ -4,6 +4,8 @@ use std::time::Duration; use crate::Connection; use crate::Error; +const SCHEMA_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + const MIGRATIONS: [&str; 7] = [ include_str!("../migrations/0001_otel_traces.sql"), include_str!("../migrations/0002_agent_traces.sql"), @@ -49,11 +51,30 @@ pub async fn ensure_schema( database: &str, trace_retention_days: u32, spend_log_retention_days: u32, +) -> Result<(), Error> { + ensure_schema_with_timeout( + client, + connection, + database, + trace_retention_days, + spend_log_retention_days, + SCHEMA_REQUEST_TIMEOUT, + ) + .await +} + +async fn ensure_schema_with_timeout( + client: &Client, + connection: &Connection, + database: &str, + trace_retention_days: u32, + spend_log_retention_days: u32, + request_timeout: Duration, ) -> Result<(), Error> { for statement in schema_statements(database, trace_retention_days, spend_log_retention_days)? { let response = client .post(connection.url().clone()) - .timeout(Duration::from_secs(10)) + .timeout(request_timeout) .body(statement) .send() .await diff --git a/litellm-rust/crates/traces/tests/admin_sql.rs b/litellm-rust/crates/traces/tests/admin_sql.rs index 20a7a229fdc..ab0eb873a28 100644 --- a/litellm-rust/crates/traces/tests/admin_sql.rs +++ b/litellm-rust/crates/traces/tests/admin_sql.rs @@ -40,7 +40,12 @@ async fn database() -> Result> { "CREATE DATABASE litellm", "CREATE TABLE litellm.otel_traces (n UInt8) ENGINE = Memory", "INSERT INTO litellm.otel_traces VALUES (1)", + "CREATE TABLE litellm.agent_traces_by_key (n UInt8) ENGINE = Memory", + "INSERT INTO litellm.agent_traces_by_key VALUES (4)", + "CREATE TABLE litellm.spend_logs (n UInt8) ENGINE = Memory", + "INSERT INTO litellm.spend_logs VALUES (3)", "CREATE TABLE litellm.private_traces (n UInt8) ENGINE = Memory", + "CREATE TABLE private_traces (n UInt8) ENGINE = Memory", ] { client .post(&admin_url) @@ -81,6 +86,15 @@ async fn admin_sql_reads_rows_with_enforced_settings( let json: Value = serde_json::from_str(&result)?; assert_eq!(json["data"][0]["answer"], 1); + let result = read( + &database.client, + &connection, + "SELECT n AS answer FROM agent_traces_by_key", + ) + .await?; + let json: Value = serde_json::from_str(&result)?; + assert_eq!(json["data"][0]["answer"], 4); + Ok(()) } diff --git a/litellm-rust/crates/traces/tests/migrations.rs b/litellm-rust/crates/traces/tests/migrations.rs index 48baaee0828..ac0266409fa 100644 --- a/litellm-rust/crates/traces/tests/migrations.rs +++ b/litellm-rust/crates/traces/tests/migrations.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeMap, time::Duration}; use litellm_http::Client; use litellm_traces::{ - Connection, Error, encode_rows, ensure_schema, execute_read, schema_statements, + Connection, Error, InsertTable, encode_rows, ensure_schema, execute_read, schema_statements, }; use rstest::{fixture, rstest}; use testcontainers_modules::{ @@ -107,7 +107,7 @@ async fn schema_supports_span_rollups_and_spend_joins( #[future(awt)] database: TestResult, ) -> TestResult { let database = database?; - let writer = Connection::writer(&database.url, "default", "")?; + let writer = Connection::writer(&database.url)?; ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; let timestamp = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64; @@ -144,7 +144,7 @@ async fn schema_supports_span_rollups_and_spend_joins( let body = read_json( &database, "SELECT toUInt32(sum(SpanCount)) AS spans, toUInt32(sum(InputTokens)) AS tokens \ - FROM trace_test.agent_traces WHERE TeamId = 'team-1' AND TraceId = 'trace-1'", + FROM trace_test.agent_traces_by_key WHERE TeamId = 'team-1' AND TraceId = 'trace-1'", ) .await?; assert_eq!( @@ -154,13 +154,92 @@ async fn schema_supports_span_rollups_and_spend_joins( Ok(()) } +#[rstest] +#[tokio::test] +async fn retried_trace_insert_does_not_inflate_rollup( + #[future(awt)] database: TestResult, +) -> TestResult { + let database = database?; + let writer = Connection::writer(&database.url)?; + ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; + let row: BTreeMap = serde_json::from_value(serde_json::json!({ + "Timestamp": time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64, + "TraceId": "retried-trace", "SpanId": "span-1", "ParentSpanId": "", + "TeamId": "team-1", "ApiKeyHash": "key-1", "SpanName": "root", "InputTokens": 7 + }))?; + for _ in 0..2 { + litellm_traces::insert_rows( + &database.client, + &writer, + "trace_test", + InsertTable::OtelTraces, + vec![row.clone()], + ) + .await?; + } + let counts = read_json( + &database, + "SELECT toUInt32(sum(SpanCount)) AS spans, toUInt32(sum(InputTokens)) AS tokens \ + FROM trace_test.agent_traces_by_key WHERE TraceId = 'retried-trace'", + ) + .await?; + assert_eq!(table_rows(&database, "otel_traces").await?, 1); + assert_eq!(counts["data"][0]["spans"], 1); + assert_eq!(counts["data"][0]["tokens"], 7); + Ok(()) +} + +#[rstest] +#[tokio::test] +async fn keyed_rollup_keeps_same_trace_ids_separate_by_api_key( + #[future(awt)] database: TestResult, +) -> TestResult { + let database = database?; + let writer = Connection::writer(&database.url)?; + ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; + let timestamp = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64; + let rows = vec![ + serde_json::from_value(serde_json::json!({ + "Timestamp": timestamp, "TraceId": "shared-id", "SpanId": "root-one", + "ParentSpanId": "", "SpanName": "root-one", "Input": "private-one", + "ResourceAttributes": {"litellm.api_key_hash": "key-one"} + }))?, + serde_json::from_value(serde_json::json!({ + "Timestamp": timestamp, "TraceId": "shared-id", "SpanId": "root-two", + "ParentSpanId": "", "SpanName": "root-two", "Input": "private-two", + "ResourceAttributes": {"litellm.api_key_hash": "key-two"} + }))?, + ]; + insert_rows(&database, "otel_traces", rows).await?; + execute_write( + &database, + "OPTIMIZE TABLE trace_test.agent_traces_by_key FINAL", + ) + .await?; + let rows = read_json( + &database, + "SELECT ApiKeyHash, any(RootInput) AS RootInput \ + FROM trace_test.agent_traces_by_key WHERE TraceId = 'shared-id' \ + GROUP BY ApiKeyHash ORDER BY ApiKeyHash", + ) + .await?; + assert_eq!( + rows["data"], + serde_json::json!([ + {"ApiKeyHash": "key-one", "RootInput": "private-one"}, + {"ApiKeyHash": "key-two", "RootInput": "private-two"} + ]) + ); + Ok(()) +} + #[rstest] #[tokio::test] async fn rollup_merges_spans_across_days_without_losing_root_fields( #[future(awt)] database: TestResult, ) -> TestResult { let database = database?; - let writer = Connection::writer(&database.url, "default", "")?; + let writer = Connection::writer(&database.url)?; ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; let day_start = time::OffsetDateTime::now_utc() .replace_time(time::Time::MIDNIGHT) @@ -179,12 +258,16 @@ async fn rollup_merges_spans_across_days_without_losing_root_fields( "ResourceAttributes": {"litellm.team_id": "team-1"} }))?; insert_rows(&database, "otel_traces", vec![child]).await?; - execute_write(&database, "OPTIMIZE TABLE trace_test.agent_traces FINAL").await?; + execute_write( + &database, + "OPTIMIZE TABLE trace_test.agent_traces_by_key FINAL", + ) + .await?; let response = read_json( &database, "SELECT count() AS rows, any(RootName) AS RootName, any(RootInput) AS RootInput, \ any(RootStatus) AS RootStatus, sum(SpanCount) AS SpanCount \ - FROM trace_test.agent_traces", + FROM trace_test.agent_traces_by_key", ) .await?; assert_eq!( @@ -203,7 +286,7 @@ async fn spend_deduplication_preserves_subsecond_requests_and_retries( #[future(awt)] database: TestResult, ) -> TestResult { let database = database?; - let writer = Connection::writer(&database.url, "default", "")?; + let writer = Connection::writer(&database.url)?; ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; let now_ms = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64 / 1_000_000; let base_start_time = now_ms / 1000 * 1000; @@ -255,7 +338,7 @@ async fn retention_changes_materialize_existing_rows_and_remain_idempotent( #[future(awt)] database: TestResult, ) -> TestResult { let database = database?; - let writer = Connection::writer(&database.url, "default", "")?; + let writer = Connection::writer(&database.url)?; ensure_schema(&database.client, &writer, "trace_test", 30, 30).await?; let old_time = time::OffsetDateTime::now_utc() - time::Duration::days(20); let old_timestamp_ns = old_time.unix_timestamp_nanos() as i64; @@ -271,6 +354,7 @@ async fn retention_changes_materialize_existing_rows_and_remain_idempotent( }))?; insert_rows(&database, "otel_traces", vec![span]).await?; insert_rows(&database, "spend_logs", vec![spend]).await?; + assert_eq!(table_rows(&database, "agent_traces_by_key").await?, 1); ensure_schema(&database.client, &writer, "trace_test", 14, 14).await?; let deadline = tokio::time::Instant::now() + Duration::from_secs(60); loop { @@ -293,10 +377,14 @@ async fn retention_changes_materialize_existing_rows_and_remain_idempotent( tokio::time::sleep(Duration::from_millis(100)).await; } execute_write(&database, "OPTIMIZE TABLE trace_test.otel_traces FINAL").await?; - execute_write(&database, "OPTIMIZE TABLE trace_test.agent_traces FINAL").await?; + execute_write( + &database, + "OPTIMIZE TABLE trace_test.agent_traces_by_key FINAL", + ) + .await?; execute_write(&database, "OPTIMIZE TABLE trace_test.spend_logs FINAL").await?; assert_eq!(table_rows(&database, "otel_traces").await?, 0); - assert_eq!(table_rows(&database, "agent_traces").await?, 0); + assert_eq!(table_rows(&database, "agent_traces_by_key").await?, 0); assert_eq!(table_rows(&database, "spend_logs").await?, 0); let mutation_count = mutation_rows(&database).await?; ensure_schema(&database.client, &writer, "trace_test", 14, 14).await?; @@ -315,9 +403,9 @@ async fn schema_statement_timeout_maps_to_transport_error() -> TestResult { }); let client = Client::no_redirect_for_test(); let url = format!("http://{address}"); - let writer = Connection::writer(&url, "default", "")?; + let writer = Connection::writer(&url)?; let result = tokio::time::timeout( - Duration::from_secs(12), + Duration::from_secs(35), ensure_schema(&client, &writer, "trace_test", 7, 14), ) .await; diff --git a/litellm-rust/crates/traces/tests/otlp.rs b/litellm-rust/crates/traces/tests/otlp.rs new file mode 100644 index 00000000000..002ba159ef9 --- /dev/null +++ b/litellm-rust/crates/traces/tests/otlp.rs @@ -0,0 +1,47 @@ +use flate2::{Compression, write::GzEncoder}; +use litellm_traces::decode_otlp; +use rstest::rstest; +use std::io::Write; + +const FIXTURE: &[u8] = include_bytes!( + "../../../../tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json" +); + +#[rstest] +#[case::json(FIXTURE, Some("application/json"), None)] +#[case::gzip_json(FIXTURE, Some("application/json"), Some("gzip"))] +fn decodes_neutral_spans( + #[case] body: &[u8], + #[case] content_type: Option<&str>, + #[case] content_encoding: Option<&str>, +) { + let payload = if content_encoding == Some("gzip") { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).expect("gzip input"); + encoder.finish().expect("gzip payload") + } else { + body.to_vec() + }; + let spans = decode_otlp(&payload, content_type, content_encoding, 8 * 1024 * 1024) + .expect("valid OTLP export"); + assert_eq!(spans.len(), 6); + assert_eq!(spans[0].trace_id, "4bad42b84e9de3ba46fc870185f8f023"); + assert_eq!(spans[0].resource_attributes["service.name"], "agent-demo"); + assert_eq!(spans[0].scope_name, "langsmith"); + assert!( + spans + .iter() + .any(|span| span.attributes.contains_key("gen_ai.prompt")) + ); +} + +#[rstest] +#[case::invalid(b"not protobuf", None, 8 * 1024 * 1024)] +#[case::too_large(FIXTURE, Some("application/json"), 1)] +fn rejects_invalid_or_oversized_payload( + #[case] body: &[u8], + #[case] content_type: Option<&str>, + #[case] limit: usize, +) { + assert!(decode_otlp(body, content_type, None, limit).is_err()); +} diff --git a/litellm/constants.py b/litellm/constants.py index 530d678457d..9af40744896 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -46,6 +46,19 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset( ) DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) +# Agent tracing / ClickHouse +CLICKHOUSE_BATCH_SIZE: Final = get_env_int("CLICKHOUSE_BATCH_SIZE", 10_000) +CLICKHOUSE_FLUSH_INTERVAL_SECONDS: Final = float(os.getenv("CLICKHOUSE_FLUSH_INTERVAL_SECONDS", "1.0")) +CLICKHOUSE_MAX_BUFFERED_ROWS: Final = get_env_int("CLICKHOUSE_MAX_BUFFERED_ROWS", 200_000) +CLICKHOUSE_MAX_RETRIES: Final = get_env_int("CLICKHOUSE_MAX_RETRIES", 3) +AGENT_TRACING_RETENTION_DAYS: Final = get_env_int("AGENT_TRACING_RETENTION_DAYS", 30) +AGENT_TRACING_SPEND_LOG_RETENTION_DAYS: Final = get_env_int("AGENT_TRACING_SPEND_LOG_RETENTION_DAYS", 90) +OTLP_MAX_BODY_BYTES: Final = get_env_int("OTLP_MAX_BODY_BYTES", 8 * 1024 * 1024) +OTLP_MAX_ATTRIBUTE_VALUE_BYTES: Final = get_env_int("OTLP_MAX_ATTRIBUTE_VALUE_BYTES", 64 * 1024) +OTLP_RETRY_AFTER_SECONDS: Final = get_env_int("OTLP_RETRY_AFTER_SECONDS", 2) +OTLP_OFFLOAD_DECODE_BYTES: Final = get_env_int("OTLP_OFFLOAD_DECODE_BYTES", 256 * 1024) +AGENT_TRACING_INPUT_PREVIEW_CHARS: Final = get_env_int("AGENT_TRACING_INPUT_PREVIEW_CHARS", 240) +AGENT_TRACING_LIST_PAGE_SIZE: Final = get_env_int("AGENT_TRACING_LIST_PAGE_SIZE", 50) DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) DEFAULT_S3_BATCH_SIZE: Final = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512)) DEFAULT_S3_MAX_CONCURRENT_UPLOADS: Final = int(os.getenv("DEFAULT_S3_MAX_CONCURRENT_UPLOADS", "16")) diff --git a/litellm/integrations/clickhouse/clickhouse_batch_logger.py b/litellm/integrations/clickhouse/clickhouse_batch_logger.py new file mode 100644 index 00000000000..fb9088f44ff --- /dev/null +++ b/litellm/integrations/clickhouse/clickhouse_batch_logger.py @@ -0,0 +1,100 @@ +""" +Shared base for everything LiteLLM writes to ClickHouse. + +Built on `CustomBatchLogger`: rows accumulate in `log_queue` and are flushed as one +gzip JSONEachRow insert, either every `CLICKHOUSE_FLUSH_INTERVAL_SECONDS` or as soon as +`batch_size` rows are queued. Subclasses only pick the table and build rows: + +- `ClickHouseSpendLogger` -> spend_logs (LiteLLM requests, via the `clickhouse` callback) +""" + +import asyncio +import os +from typing import Any, ClassVar + +from litellm._logging import verbose_logger +from litellm.constants import ( + CLICKHOUSE_BATCH_SIZE, + CLICKHOUSE_FLUSH_INTERVAL_SECONDS, + CLICKHOUSE_MAX_BUFFERED_ROWS, + CLICKHOUSE_MAX_RETRIES, +) +from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.rust_bridge.traces import TraceStorage + + +def clickhouse_storage_from_env() -> TraceStorage: + return TraceStorage( + database=os.getenv("CLICKHOUSE_DATABASE", "litellm"), + url=os.getenv("CLICKHOUSE_URL", ""), + ) + + +class ClickHouseBatchLogger(CustomBatchLogger): + table: ClassVar[str] + + def __init__(self, storage: TraceStorage | None = None) -> None: + self.storage = storage or clickhouse_storage_from_env() + self.rows_written = 0 + self.rows_dropped = 0 + self._failed_attempts = 0 + super().__init__( + flush_lock=asyncio.Lock(), + batch_size=CLICKHOUSE_BATCH_SIZE, + flush_interval=CLICKHOUSE_FLUSH_INTERVAL_SECONDS, + ) + try: + asyncio.get_running_loop().create_task(self.periodic_flush()) + except RuntimeError: # no loop yet (e.g. sync config load); proxy startup calls start() + pass + + def start(self) -> None: + asyncio.get_running_loop().create_task(self.periodic_flush()) + + def is_full(self) -> bool: + """Backpressure signal: producers should reject (429) instead of enqueueing.""" + return len(self.log_queue) >= CLICKHOUSE_MAX_BUFFERED_ROWS + + def enqueue(self, rows: list[dict[str, Any]]) -> None: + """Never awaits ClickHouse. Kicks off an early flush once a full batch is queued.""" + self.log_queue.extend(rows) + if len(self.log_queue) >= self.batch_size: + asyncio.get_running_loop().create_task(self.flush_queue()) + + async def flush_queue(self) -> None: + # Swap the queue under the lock so rows enqueued during the insert are kept. + if self.flush_lock is None: + return + async with self.flush_lock: + while self.log_queue: + batch = self.log_queue[: self.batch_size] + self.log_queue = self.log_queue[len(batch) :] + if not await self._insert(batch): + break + + async def async_send_batch(self) -> None: + await self.flush_queue() + + async def _insert(self, batch: list[dict[str, Any]]) -> bool: + try: + await self.storage.insert_rows(self.table, batch) + self.rows_written += len(batch) + self._failed_attempts = 0 + return True + except Exception as e: + self._failed_attempts += 1 + if self._failed_attempts >= CLICKHOUSE_MAX_RETRIES: + self.rows_dropped += len(batch) + self._failed_attempts = 0 + verbose_logger.error( + "ClickHouse: dropped %s rows for %s after %s attempts: %s", + len(batch), + self.table, + CLICKHOUSE_MAX_RETRIES, + e, + ) + else: + # put it back; the next periodic flush retries it + self.log_queue = batch + self.log_queue + verbose_logger.warning("ClickHouse: insert into %s failed, will retry: %s", self.table, e) + return False diff --git a/litellm/integrations/clickhouse/schema.py b/litellm/integrations/clickhouse/schema.py new file mode 100644 index 00000000000..6bec35c5630 --- /dev/null +++ b/litellm/integrations/clickhouse/schema.py @@ -0,0 +1,11 @@ +from typing import Final + +from litellm.rust_bridge.traces import TraceStorage + +OTEL_TRACES_TABLE: Final = "otel_traces" +AGENT_TRACES_BY_KEY_TABLE: Final = "agent_traces_by_key" +SPEND_LOGS_TABLE: Final = "spend_logs" + + +async def ensure_schema(storage: TraceStorage, trace_retention_days: int, spend_log_retention_days: int) -> None: + await storage.ensure_schema(trace_retention_days, spend_log_retention_days) diff --git a/litellm/integrations/custom_batch_logger.py b/litellm/integrations/custom_batch_logger.py index bfc78b93715..2e1cf291716 100644 --- a/litellm/integrations/custom_batch_logger.py +++ b/litellm/integrations/custom_batch_logger.py @@ -27,7 +27,7 @@ class CustomBatchLogger(CustomLogger): self, flush_lock: asyncio.Lock | None = None, batch_size: int | None = None, - flush_interval: int | None = None, + flush_interval: float | None = None, max_queue_size: int | None = None, **kwargs, ) -> None: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9c7dece6894..a68462313f1 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -520,6 +520,10 @@ class LiteLLMRoutes(enum.Enum): "/v1/rag/ingest", "/rag/query", "/v1/rag/query", + # agent tracing: OTLP ingest + reads (scoped to the caller's team in the handler) + "/v1/traces", + "/v1/traces/{trace_id}", + "/v1/traces/{trace_id}/spans/{span_id}", ] anthropic_routes = [ diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index f436bc96dbd..ac757f5f6a7 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -21,6 +21,9 @@ from litellm.proxy.common_utils.callback_utils import ( from litellm.types.router import Deployment _FORM_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-www-form-urlencoded", "multipart/form-data"}) +# Binary bodies (e.g. OTLP trace exports on POST /v1/traces) are not JSON: arbitrary bytes used to +# hit the JSON surrogate-repair path and fail auth with a 400. JSON under these types still parses. +_BINARY_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-protobuf", "application/protobuf"}) _ANNOTATION_QUALIFIERS: Final[frozenset[object]] = frozenset({Annotated, NotRequired, ReadOnly, Required}) @@ -153,6 +156,17 @@ def coerce_numeric_form_fields( } +def _parse_binary_body(body: bytes) -> dict: + """JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}.""" + try: + parsed: Final = orjson.loads(body) + if isinstance(parsed, dict): + return parsed + except orjson.JSONDecodeError: + pass + return {} # mutable-ok: auth parser returns a fresh dict per request + + async def _read_request_body(request: Request | None) -> dict: """ Safely read the request body and parse it as JSON. @@ -175,7 +189,13 @@ async def _read_request_body(request: Request | None) -> dict: _request_headers: Final[dict] = _safe_get_request_headers(request=request) content_type: Final = _request_headers.get("content-type", "") - if _is_form_content_type(content_type): + if _normalize_media_type(content_type) in _BINARY_CONTENT_TYPES or ( + request.scope.get("path") == "/v1/traces" + and request.scope.get("method") == "POST" + and _request_headers.get("content-encoding", "").lower() == "gzip" + ): + parsed_body = _parse_binary_body(await request.body()) + elif _is_form_content_type(content_type): try: form_data: Final = await request.form() except Exception as e: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7200407b64c..6ae5cb6a0c0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -714,6 +714,7 @@ try: except ImportError: build_billing_metrics_recorder = None shutdown_billing_metrics_recorder = None +from litellm.proxy import tracing_endpoints from litellm.proxy.middleware.admission_control_middleware import ( AdmissionControlMiddleware, admission_control_state, @@ -845,6 +846,7 @@ from litellm.secret_managers.main import ( secret_manager_would_be_consulted, str_to_bool, ) +from litellm.tracing import TraceReceiver from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs from litellm.types.llms.anthropic import ( AnthropicMessagesRequest, @@ -1521,6 +1523,9 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: _tagged.strategy._state_loaded = True asyncio.create_task(_adaptive_router_flusher_loop()) + ## [Optional] Initialize agent tracing + asyncio.create_task(ProxyStartupEvent.init_tracing(general_settings)) + ## [Optional] Initialize dd tracer ProxyStartupEvent._init_dd_tracer() @@ -11310,6 +11315,28 @@ class ProxyStartupEvent: ) return connected_client + @classmethod + async def init_tracing(cls, general_settings: dict) -> None: + """ + Enable agent tracing (`POST/GET /v1/traces`) when configured: + + general_settings: + tracing: + store: clickhouse # CLICKHOUSE_URL / _USER / _PASSWORD / _DATABASE + """ + settings: Final = general_settings.get("tracing") + if not isinstance(settings, dict) or settings.get("store") != "clickhouse": + return + try: + tracing: Final = TraceReceiver.from_env() + await tracing.start() + except (KeyError, OSError, RuntimeError, ValueError) as error: + tracing_endpoints.receiver = None + verbose_proxy_logger.warning("Agent tracing unavailable: %s", error) + return + tracing_endpoints.receiver = tracing + verbose_proxy_logger.info("Agent tracing enabled (store=clickhouse)") + @classmethod def _init_dd_tracer(cls): """ @@ -19849,6 +19876,7 @@ app.include_router(rag_router) app.include_router(video_router) app.include_router(container_router) app.include_router(search_router) +app.include_router(tracing_endpoints.router) app.include_router(image_router) app.include_router(fine_tuning_router) app.include_router(credential_router) diff --git a/litellm/proxy/tracing_endpoints.py b/litellm/proxy/tracing_endpoints.py new file mode 100644 index 00000000000..06dbf359ba9 --- /dev/null +++ b/litellm/proxy/tracing_endpoints.py @@ -0,0 +1,141 @@ +""" +Agent tracing endpoints. Thin wrappers over `TraceReceiver`: auth -> tenant/scope -> one call. + +POST /v1/traces OTLP/HTTP trace export (protobuf or JSON) +GET /v1/traces TracePage +GET /v1/traces/{trace_id} Trace +GET /v1/traces/{trace_id}/spans/{span_id} SpanDetail +""" + +import time +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response + +from litellm.constants import OTLP_MAX_BODY_BYTES, OTLP_RETRY_AFTER_SECONDS +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.tracing import ( + Tenant, + TraceReceiver, + TracingPayloadTooLargeError, +) +from litellm.tracing.decode import InvalidOTLPPayloadError, encode_otlp_response +from litellm.tracing.types import SpanDetail, Trace, TracePage, TraceScope + +router = APIRouter(tags=["agent tracing"]) # mutable-ok: FastAPI copies the mutable tags list + +MS_PER_DAY: Final = 24 * 60 * 60 * 1000 +_ADMIN_ROLES: Final = (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) + +receiver: TraceReceiver | None = None + + +def get_receiver() -> TraceReceiver: + if receiver is None: + raise HTTPException( + status_code=501, + detail="Agent tracing is not enabled. Set `tracing:` in general_settings and CLICKHOUSE_URL.", + ) + return receiver + + +def tenant_for(user_api_key_dict: UserAPIKeyAuth) -> Tenant: + return Tenant( + team_id=user_api_key_dict.team_id or "", + api_key_hash=user_api_key_dict.token or "", + org_id=user_api_key_dict.org_id or "", + ) + + +def scope_for(user_api_key_dict: UserAPIKeyAuth) -> TraceScope: + """Admins see everything; team members see their team; team-less keys see their own traces.""" + if user_api_key_dict.user_role in _ADMIN_ROLES: + return TraceScope(team_ids=(), api_key_hash="") + if user_api_key_dict.team_id: + return TraceScope(team_ids=(user_api_key_dict.team_id,), api_key_hash="") + if not user_api_key_dict.token: + raise HTTPException(status_code=403, detail="Not allowed to view agent traces") + return TraceScope(team_ids=("",), api_key_hash=user_api_key_dict.token) + + +async def _read_otlp_body(request: Request) -> bytes: + body: Final = bytearray() + async for chunk in request.stream(): + if len(body) + len(chunk) > OTLP_MAX_BODY_BYTES: + raise TracingPayloadTooLargeError(f"OTLP body exceeds {OTLP_MAX_BODY_BYTES} bytes") + body.extend(chunk) + return bytes(body) + + +@router.post("/v1/traces", include_in_schema=False) +async def ingest_otlp_traces( + request: Request, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY: + raise HTTPException(status_code=403, detail="Not allowed to ingest agent traces") + tracing: Final = get_receiver() + content_type: Final = request.headers.get("content-type") + try: + await tracing.ingest( + body=await _read_otlp_body(request), + content_type=content_type, + content_encoding=request.headers.get("content-encoding"), + tenant=tenant_for(user_api_key_dict), + ) + except TracingPayloadTooLargeError as e: + raise HTTPException(status_code=413, detail=str(e)) + except InvalidOTLPPayloadError as error: + raise HTTPException(status_code=400, detail=str(error)) from error + except RuntimeError: + raise HTTPException( + status_code=503, + headers={"Retry-After": str(OTLP_RETRY_AFTER_SECONDS)}, # mutable-ok: FastAPI requires dict headers + ) + body, media_type = encode_otlp_response(content_type) + return Response(content=body, media_type=media_type) + + +@router.get("/v1/traces", response_model=None) +async def list_agent_traces( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_ms: Annotated[int | None, Query(description="Window start, unix ms. Default: 24h ago")] = None, + end_ms: Annotated[int | None, Query(description="Window end, unix ms. Default: now")] = None, + cursor: Annotated[str | None, Query()] = None, +) -> TracePage: + now_ms: Final = int(time.time() * 1000) + try: + return await get_receiver().list_traces( + scope=scope_for(user_api_key_dict), + start_ms=start_ms if start_ms is not None else now_ms - MS_PER_DAY, + end_ms=end_ms if end_ms is not None else now_ms, + cursor=cursor, + ) + except ValueError as error: + raise HTTPException(status_code=400, detail=str(error)) from error + + +@router.get("/v1/traces/{trace_id}", response_model=None) +async def get_agent_trace( + trace_id: str, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + trace_ref: Annotated[str, Query()] = "", +) -> Trace: + trace: Final = await get_receiver().get_trace(trace_id, scope_for(user_api_key_dict), trace_ref) + if trace is None: + raise HTTPException(status_code=404, detail=f"Trace {trace_id} not found") + return trace + + +@router.get("/v1/traces/{trace_id}/spans/{span_id}", response_model=None) +async def get_agent_trace_span( + trace_id: str, + span_id: str, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + trace_ref: Annotated[str, Query()] = "", +) -> SpanDetail: + span: Final = await get_receiver().get_span(trace_id, span_id, scope_for(user_api_key_dict), trace_ref) + if span is None: + raise HTTPException(status_code=404, detail=f"Span {span_id} not found") + return span diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 5d2af93a9bd..30d4bbfb68e 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -11,6 +11,7 @@ from litellm.rust_bridge.embeddings.entrypoints import LiteLLMEmbeddingRequest from litellm.rust_bridge.messages.entrypoints import LiteLLMMessagesRequest from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest from litellm.rust_bridge.responses.entrypoints import LiteLLMResponsesRequest +from litellm.rust_bridge.traces import DecodedSpan from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import EmbeddingResponse, ModelResponse @@ -20,18 +21,16 @@ class RustUpstreamError(Exception): ... class ForkedAfterNativeRuntimeStarted(RuntimeError): ... class ProcessReservedForForking(RuntimeError): ... -def trace_encode_rows(rows: Sequence[Mapping[str, JsonValue]]) -> str: ... -def trace_ensure_schema( - url: str, database: str, user: str, password: str, trace_retention_days: int, spend_log_retention_days: int -) -> Future[None]: ... -def trace_query( - url: str, - database: str, - user: str, - password: str, - sql: str, - parameters: Mapping[str, str | int | Sequence[str]], -) -> Future[str]: ... +def trace_decode_otlp( + body: bytes, content_type: str | None, content_encoding: str | None, max_decompressed_bytes: int +) -> list[DecodedSpan]: ... + +@final +class NativeTraceStorage: + def __new__(cls, database: str, url: str, reader_url: str | None = None) -> NativeTraceStorage: ... + def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> Future[None]: ... + def insert_rows(self, table: str, rows: Sequence[Mapping[str, JsonValue]]) -> Future[None]: ... + def query(self, sql: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Future[str]: ... @final class NativeDiagnosticProcessor: @@ -327,6 +326,7 @@ __all__ = [ "ForkedAfterNativeRuntimeStarted", "HuggingFaceEncoding", "NativeDiagnosticProcessor", + "NativeTraceStorage", "ProcessReservedForForking", "ResponsesWebSocketConnection", "RustBridgeDeclined", @@ -351,9 +351,7 @@ __all__ = [ "process_state_started", "reserve_process_for_forking", "responses", - "trace_encode_rows", - "trace_ensure_schema", - "trace_query", + "trace_decode_otlp", "transcription", ] diff --git a/litellm/rust_bridge/traces.py b/litellm/rust_bridge/traces.py index cb4fd4a5bd3..a0b010caea0 100644 --- a/litellm/rust_bridge/traces.py +++ b/litellm/rust_bridge/traces.py @@ -1,33 +1,56 @@ from collections.abc import Awaitable, Mapping, Sequence -from typing import Final, Protocol, cast +from types import MappingProxyType +from typing import Final, Protocol, TypedDict, cast -from pydantic import BaseModel, ConfigDict, JsonValue +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter +from typing_extensions import ReadOnly from litellm.rust_bridge.loader import get_native_bridge +class DecodedEvent(TypedDict): + name: ReadOnly[str] + attributes: ReadOnly[dict[str, str]] + + +class DecodedSpan(TypedDict): + trace_id: ReadOnly[str] + span_id: ReadOnly[str] + parent_span_id: ReadOnly[str] + trace_state: ReadOnly[str] + name: ReadOnly[str] + kind: ReadOnly[str] + resource_attributes: ReadOnly[dict[str, str]] + scope_name: ReadOnly[str] + scope_version: ReadOnly[str] + attributes: ReadOnly[dict[str, str]] + start_ns: ReadOnly[int] + end_ns: ReadOnly[int] + status_code: ReadOnly[str] + status_message: ReadOnly[str] + events: ReadOnly[list[DecodedEvent]] + + +class NativeStore(Protocol): + def __init__(self, database: str, url: str, reader_url: str | None = None) -> None: ... + + def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> Awaitable[None]: ... + + def insert_rows(self, table: str, rows: Sequence[Mapping[str, JsonValue]]) -> Awaitable[None]: ... + + def query(self, sql: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Awaitable[str]: ... + + class NativeTraces(Protocol): - def trace_encode_rows(self, rows: Sequence[Mapping[str, JsonValue]]) -> str: ... + NativeTraceStorage: type[NativeStore] - def trace_ensure_schema( + def trace_decode_otlp( self, - url: str, - database: str, - user: str, - password: str, - trace_retention_days: int, - spend_log_retention_days: int, - ) -> Awaitable[None]: ... - - def trace_query( - self, - url: str, - database: str, - user: str, - password: str, - sql: str, - parameters: Mapping[str, str | int | Sequence[str]], - ) -> Awaitable[str]: ... + body: bytes, + content_type: str | None, + content_encoding: str | None, + max_decompressed_bytes: int, + ) -> list[DecodedSpan]: ... class QueryResponse(BaseModel): @@ -35,6 +58,10 @@ class QueryResponse(BaseModel): data: list[dict[str, JsonValue]] +INSERT_ROWS: Final = TypeAdapter(list[dict[str, JsonValue]]) +QUERY_PARAMETERS: Final = TypeAdapter(dict[str, str | int | list[str]]) + + def _native() -> NativeTraces: native: Final = get_native_bridge() if native is None: @@ -42,28 +69,24 @@ def _native() -> NativeTraces: return cast(NativeTraces, native) # cast-ok: the native extension is validated against this protocol at call sites -async def ensure_schema( - url: str, - database: str, - user: str, - password: str, - trace_retention_days: int, - spend_log_retention_days: int, -) -> None: - await _native().trace_ensure_schema(url, database, user, password, trace_retention_days, spend_log_retention_days) +def decode_otlp( + body: bytes, content_type: str | None, content_encoding: str | None, max_decompressed_bytes: int +) -> list[DecodedSpan]: + return _native().trace_decode_otlp(body, content_type, content_encoding, max_decompressed_bytes) -async def query( - url: str, - database: str, - user: str, - password: str, - sql: str, - parameters: Mapping[str, str | int | Sequence[str]], -) -> list[dict[str, JsonValue]]: - result: Final = await _native().trace_query(url, database, user, password, sql, parameters) - return QueryResponse.model_validate_json(result).data +class TraceStorage: + def __init__(self, database: str, url: str, reader_url: str | None = None) -> None: + self._native: Final = _native().NativeTraceStorage(database, url, reader_url) + async def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> None: + await self._native.ensure_schema(trace_retention_days, spend_log_retention_days) -def encode_rows(rows: Sequence[Mapping[str, JsonValue]]) -> bytes: - return _native().trace_encode_rows(rows).encode("utf-8") + async def insert_rows(self, table: str, rows: Sequence[Mapping[str, object]]) -> None: + await self._native.insert_rows(table, INSERT_ROWS.validate_python(rows)) + + async def query(self, sql: str, parameters: Mapping[str, object] | None = None) -> list[dict[str, JsonValue]]: + result: Final = await self._native.query( + sql, QUERY_PARAMETERS.validate_python(parameters or MappingProxyType({})) + ) + return QueryResponse.model_validate_json(result).data diff --git a/litellm/tracing/AGENTS.md b/litellm/tracing/AGENTS.md new file mode 100644 index 00000000000..f69866c0419 --- /dev/null +++ b/litellm/tracing/AGENTS.md @@ -0,0 +1,6 @@ +- Python owns tracing endpoints, authenticated tenant scope, framework normalization and API response shaping +- Trace ingestion awaits `TraceStorage.insert_rows` before returning success; propagate storage failures so OTLP exporters can retry +- Spend logging keeps its separate batch queue in `litellm/integrations/clickhouse` +- Use `litellm.rust_bridge.traces.TraceStorage` for ClickHouse; keep schema, SQL, encoding and transport in `litellm-traces` +- Derive tenant fields from authentication and overwrite matching fields supplied by the exporter +- Test confirmed writes, failures, tenant isolation and read behavior through public functions diff --git a/litellm/tracing/__init__.py b/litellm/tracing/__init__.py new file mode 100644 index 00000000000..681100ed76a --- /dev/null +++ b/litellm/tracing/__init__.py @@ -0,0 +1,16 @@ +""" +LiteLLM agent tracing: OTLP traces from agents, joined to LiteLLM spend logs, in ClickHouse. + +""" + +from litellm.tracing.receiver import ( + Tenant, + TraceReceiver, + TracingPayloadTooLargeError, +) + +__all__ = ( + "Tenant", + "TraceReceiver", + "TracingPayloadTooLargeError", +) diff --git a/litellm/tracing/decode.py b/litellm/tracing/decode.py new file mode 100644 index 00000000000..60ae7732444 --- /dev/null +++ b/litellm/tracing/decode.py @@ -0,0 +1,284 @@ +""" +OTLP/HTTP trace export -> `SpanRow`s. + +Pure functions, no I/O. Two steps: +1. `decode_otlp()` protobuf / JSON / gzip `ExportTraceServiceRequest` -> flat spans +2. `normalize()` framework conventions -> LiteLLM columns (type, agent, input/output, + LiteLLM request id). Supported: LangSmith (LangChain, LangGraph, + Deep Agents), OTEL GenAI semconv, OpenInference. +""" + +import json +from collections.abc import Callable, Mapping +from types import MappingProxyType +from typing import Any, Final + +from litellm.constants import OTLP_MAX_ATTRIBUTE_VALUE_BYTES, OTLP_MAX_BODY_BYTES +from litellm.rust_bridge.traces import DecodedSpan +from litellm.rust_bridge.traces import decode_otlp as native_decode_otlp +from litellm.tracing.types import SpanRow, SpanType + +# attributes whose content we lift into Input/Output and drop from SpanAttributes +_HEAVY_ATTRIBUTES: Final = frozenset( + { + "gen_ai.prompt", + "gen_ai.completion", + "gen_ai.tool.definitions", + "gen_ai.input.messages", + "gen_ai.output.messages", + "input.value", + "output.value", + } +) +# LangChain / Deep Agents middleware wrappers: real spans, but noise in the UI +_FRAMEWORK_SUFFIXES: Final = ( + ".wrap_model_call", + ".wrap_tool_call", + ".before_agent", + ".after_agent", + ".before_model", + ".after_model", +) +_LLM_OPERATIONS: Final = frozenset({"chat", "text_completion", "generate_content"}) +_LC_ROLES: Final = MappingProxyType({"human": "user", "ai": "assistant", "system": "system", "tool": "tool"}) +_OPENINFERENCE_TYPES: Final[Mapping[str, SpanType]] = MappingProxyType({"AGENT": "agent", "LLM": "llm", "TOOL": "tool"}) + + +class InvalidOTLPPayloadError(ValueError): + pass + + +class OTLPPayloadTooLargeError(OverflowError): + pass + + +# ---------------------------------------------------------------- decode + + +def _truncate(value: str) -> str: + size = len(value.encode("utf-8")) + if size <= OTLP_MAX_ATTRIBUTE_VALUE_BYTES: + return value + kept = value.encode("utf-8")[:OTLP_MAX_ATTRIBUTE_VALUE_BYTES].decode("utf-8", "ignore") + return f"{kept}…[truncated {size - OTLP_MAX_ATTRIBUTE_VALUE_BYTES} bytes]" + + +def decode_otlp( + body: bytes, content_type: str | None = None, content_encoding: str | None = None +) -> tuple[SpanRow, ...]: + """Decode an OTLP trace export and normalize every span.""" + try: + spans: Final = native_decode_otlp(body, content_type, content_encoding, OTLP_MAX_BODY_BYTES) + except OverflowError as error: + raise OTLPPayloadTooLargeError(str(error)) from error + except ValueError as error: + raise InvalidOTLPPayloadError(str(error)) from error + return tuple(_span_row(span) for span in spans) + + +def _exception_message(span: DecodedSpan) -> str: + """`span.record_exception()` writes an `exception` event; surface it when status.message is empty.""" + for event in span["events"]: + if event["name"] == "exception": + attributes = event["attributes"] + return attributes.get("exception.message") or attributes.get("exception.type", "") + return "" + + +def _span_row(span: DecodedSpan) -> SpanRow: + attributes = span["attributes"] + resource = span["resource_attributes"] + row = SpanRow( + Timestamp=span["start_ns"], + TraceId=span["trace_id"], + SpanId=span["span_id"], + ParentSpanId=span["parent_span_id"], + TraceState=span["trace_state"], + SpanName=span["name"], + SpanKind=span["kind"], + ServiceName=resource.get("service.name", ""), + ResourceAttributes=resource, + ScopeName=span["scope_name"], + ScopeVersion=span["scope_version"], + SpanAttributes=attributes, + Duration=max(span["end_ns"] - span["start_ns"], 0), + StatusCode=span["status_code"], + StatusMessage=span["status_message"] or _exception_message(span), + TeamId="", + ApiKeyHash="", + ObservationType="chain", + AgentName="", + LiteLLMRequestId="", + Model="", + InputTokens=0, + OutputTokens=0, + Input="", + Output="", + ) + normalize(row, attributes) + row["SpanAttributes"] = { # mutable-ok: the Rust JSON bridge requires a plain dict for span attributes + k: _truncate(v) for k, v in attributes.items() if k not in _HEAVY_ATTRIBUTES + } + row["Input"], row["Output"] = _truncate(row["Input"]), _truncate(row["Output"]) + return row + + +# ---------------------------------------------------------------- normalize + + +def _loads(value: str) -> object: + try: + return json.loads(value) + except (ValueError, TypeError): + return None + + +def _lc_message(message: Mapping[str, Any]) -> dict[str, Any]: + """LangChain serialized message (or plain {role, content}) -> {role, content, tool_calls?}.""" + kwargs = message.get("kwargs", message) + role = _LC_ROLES.get(kwargs.get("type") or kwargs.get("role"), kwargs.get("role") or kwargs.get("type") or "") + content = kwargs.get("content", "") + out: dict[str, Any] = { # mutable-ok: the framework message is built for JSON serialization + "role": role, + "content": content if isinstance(content, str) else json.dumps(content), + } + if kwargs.get("tool_calls"): + out["tool_calls"] = tuple( + {"name": t.get("name"), "args": t.get("args")} # mutable-ok: JSON tool calls need object payloads + for t in kwargs["tool_calls"] + ) + if role == "tool" and kwargs.get("name"): + out["name"] = kwargs["name"] + return out + + +def _langsmith_type(row: SpanRow, attributes: Mapping[str, str]) -> SpanType: + kind = attributes.get("langsmith.span.kind", "chain") + name = row["SpanName"] + if not row["ParentSpanId"] or name == attributes.get("langsmith.metadata.lc_agent_name"): + return "agent" + if kind in ("llm", "tool"): + return kind + if name.endswith(_FRAMEWORK_SUFFIXES): + return "framework" + return "chain" + + +def _langsmith_io(row: SpanRow, attributes: Mapping[str, str]) -> None: + prompt = _loads(attributes.get("gen_ai.prompt", "")) + completion = _loads(attributes.get("gen_ai.completion", "")) + prompt_payload = prompt if isinstance(prompt, dict) else MappingProxyType({}) + if row["ObservationType"] == "llm" and isinstance(completion, dict): + messages = prompt_payload.get("messages") or ((),) + batch = messages[0] if messages and isinstance(messages[0], list) else messages + row["Input"] = ( + json.dumps(tuple(_lc_message(m) for m in batch if isinstance(m, dict))) + if isinstance(batch, (list, tuple)) + else "" + ) + generations: Final = completion.get("generations") + first: Final = generations[0] if isinstance(generations, list) and generations else None + item: Final = first[0] if isinstance(first, list) and first else None + message: Final = item.get("message") if isinstance(item, dict) else None + generation: Final = message.get("kwargs") if isinstance(message, dict) else None + if isinstance(generation, dict): + row["Output"] = json.dumps(_lc_message(generation)) + metadata: Final = generation.get("response_metadata") + row["LiteLLMRequestId"] = metadata.get("id", "") if isinstance(metadata, dict) else "" + else: + row["Output"] = attributes.get("gen_ai.completion", "") + return + if row["ObservationType"] == "tool": + output = completion.get("output", completion) if isinstance(completion, dict) else completion + if isinstance(output, dict) and "update" in output: # LangGraph Command, e.g. Deep Agents `task` + update: Final = output.get("update") + update_messages = update.get("messages") or () if isinstance(update, dict) else () + output = update_messages[-1] if update_messages else output + if isinstance(output, dict): + output = output.get("content", output) + row["Input"] = attributes.get("gen_ai.prompt", "") + row["Output"] = output if isinstance(output, str) else json.dumps(output) + return + if row["ObservationType"] == "agent": + input_messages = prompt.get("messages") if isinstance(prompt, dict) else None + output_messages = completion.get("messages") if isinstance(completion, dict) else None + # agents built with @traceable take arbitrary args, not a message list: keep the raw payload then + row["Input"] = ( + json.dumps(tuple(_lc_message(m) for m in input_messages if isinstance(m, dict))) + if input_messages + else attributes.get("gen_ai.prompt", "") + ) + row["Output"] = ( + json.dumps(_lc_message(output_messages[-1])) + if output_messages and isinstance(output_messages[-1], dict) + else attributes.get("gen_ai.completion", "") + ) + return + row["Input"] = attributes.get("gen_ai.prompt", "") + row["Output"] = attributes.get("gen_ai.completion", "") + + +def normalize_langsmith(row: SpanRow, attributes: Mapping[str, str]) -> None: + row["ObservationType"] = _langsmith_type(row, attributes) + row["AgentName"] = attributes.get("langsmith.metadata.lc_agent_name", "") + row["Model"] = attributes.get("gen_ai.request.model", "") + _langsmith_io(row, attributes) + + +def normalize_genai(row: SpanRow, attributes: Mapping[str, str]) -> None: + operation = attributes.get("gen_ai.operation.name", "") + if operation == "invoke_agent" or not row["ParentSpanId"]: + row["ObservationType"] = "agent" + elif operation in _LLM_OPERATIONS: + row["ObservationType"] = "llm" + elif operation == "execute_tool": + row["ObservationType"] = "tool" + row["AgentName"] = attributes.get("gen_ai.agent.name", "") + row["Model"] = attributes.get("gen_ai.request.model") or attributes.get("gen_ai.response.model", "") + row["LiteLLMRequestId"] = attributes.get("gen_ai.response.id", "") + row["Input"] = attributes.get("gen_ai.input.messages") or attributes.get("gen_ai.tool.call.arguments", "") + row["Output"] = attributes.get("gen_ai.output.messages") or attributes.get("gen_ai.tool.call.result", "") + + +def normalize_openinference(row: SpanRow, attributes: Mapping[str, str]) -> None: + kind = attributes.get("openinference.span.kind", "").upper() + row["ObservationType"] = _OPENINFERENCE_TYPES.get(kind, "agent" if not row["ParentSpanId"] else "chain") + row["AgentName"] = attributes.get("agent.name", "") + row["Model"] = attributes.get("llm.model_name", "") + row["Input"] = attributes.get("input.value", "") + row["Output"] = attributes.get("output.value", "") + row["InputTokens"] = _to_int(attributes.get("llm.token_count.prompt")) + row["OutputTokens"] = _to_int(attributes.get("llm.token_count.completion")) + + +def _set_tokens(row: SpanRow, attributes: Mapping[str, str]) -> None: + row["InputTokens"] = _to_int(attributes.get("gen_ai.usage.input_tokens")) + row["OutputTokens"] = _to_int(attributes.get("gen_ai.usage.output_tokens")) + + +def _to_int(value: str | None) -> int: + try: + return int(value) if value else 0 + except ValueError: + return 0 + + +def select_normalizer(scope_name: str, attributes: Mapping[str, str]) -> Callable[[SpanRow, Mapping[str, str]], None]: + if scope_name == "langsmith" or "langsmith.span.kind" in attributes: + return normalize_langsmith + if "openinference.span.kind" in attributes: + return normalize_openinference + return normalize_genai + + +def normalize(row: SpanRow, attributes: Mapping[str, str]) -> None: + select_normalizer(row["ScopeName"], attributes)(row, attributes) + if not row["InputTokens"] and not row["OutputTokens"]: + _set_tokens(row, attributes) + + +def encode_otlp_response(content_type: str | None) -> tuple[bytes, str]: + """Empty ExportTraceServiceResponse in the caller's encoding.""" + if content_type and "json" in content_type: + return b"{}", "application/json" + return b"", "application/x-protobuf" diff --git a/litellm/tracing/receiver.py b/litellm/tracing/receiver.py new file mode 100644 index 00000000000..be9641a602b --- /dev/null +++ b/litellm/tracing/receiver.py @@ -0,0 +1,120 @@ +""" +`TraceReceiver`: the one entry point for agent tracing. + + tracing = TraceReceiver.from_env() # or TraceReceiver(store=...) + await tracing.start() # create tables if missing + + tracing.ingest(otlp_body, content_type, content_encoding, tenant) # POST /v1/traces + await tracing.list_traces(scope, start_ms, end_ms, cursor) # GET /v1/traces + await tracing.get_trace(trace_id, scope) # GET /v1/traces/{id} + await tracing.get_span(trace_id, span_id, scope) # GET /v1/traces/{id}/spans/{span_id} + +The proxy endpoints are thin wrappers: auth -> build tenant/scope -> call one method. +""" + +import asyncio +import os +from typing import Final + +from litellm.constants import ( + AGENT_TRACING_RETENTION_DAYS, + AGENT_TRACING_SPEND_LOG_RETENTION_DAYS, + OTLP_MAX_BODY_BYTES, + OTLP_OFFLOAD_DECODE_BYTES, +) +from litellm.integrations.clickhouse.schema import ensure_schema +from litellm.rust_bridge.traces import TraceStorage +from litellm.tracing.decode import OTLPPayloadTooLargeError, decode_otlp +from litellm.tracing.store import ClickHouseTraceStore +from litellm.tracing.types import ( + SpanDetail, + SpanRow, + Trace, + TracePage, + TraceScope, +) + + +class TracingPayloadTooLargeError(Exception): + pass + + +class Tenant: + """Who sent the spans. Always taken from auth, never from span attributes.""" + + def __init__(self, team_id: str, api_key_hash: str, org_id: str = "") -> None: + self.team_id = team_id + self.api_key_hash = api_key_hash + self.org_id = org_id + + def stamp(self, row: SpanRow) -> SpanRow: + row["TeamId"] = self.team_id + row["ApiKeyHash"] = self.api_key_hash + row["ResourceAttributes"] = { # mutable-ok: the Rust JSON bridge requires a plain dict + **row["ResourceAttributes"], + "litellm.team_id": self.team_id, + "litellm.api_key_hash": self.api_key_hash, + "litellm.org_id": self.org_id, + } + return row + + +class TraceReceiver: + def __init__(self, store: ClickHouseTraceStore) -> None: + self.store = store + + @classmethod + def from_env(cls) -> "TraceReceiver": + return cls( + store=ClickHouseTraceStore( + TraceStorage( + database=os.getenv("CLICKHOUSE_DATABASE", "litellm"), + url=os.environ["CLICKHOUSE_URL"], + reader_url=os.environ["CLICKHOUSE_READER_URL"], + ) + ) + ) + + async def start(self) -> None: + await ensure_schema( + self.store.storage, + trace_retention_days=AGENT_TRACING_RETENTION_DAYS, + spend_log_retention_days=AGENT_TRACING_SPEND_LOG_RETENTION_DAYS, + ) + + # ------------------------------------------------------------ write + + async def ingest( + self, + body: bytes, + content_type: str | None, + content_encoding: str | None, + tenant: Tenant, + ) -> int: + """Decode an OTLP trace export and store its authenticated spans.""" + if len(body) > OTLP_MAX_BODY_BYTES: + raise TracingPayloadTooLargeError(f"OTLP body exceeds {OTLP_MAX_BODY_BYTES} bytes") + try: + rows: Final = ( + await asyncio.to_thread(decode_otlp, body, content_type, content_encoding) + if len(body) > OTLP_OFFLOAD_DECODE_BYTES + else decode_otlp(body, content_type, content_encoding) + ) + except OTLPPayloadTooLargeError as error: + raise TracingPayloadTooLargeError(str(error)) from error + try: + await self.store.insert_spans(tuple(tenant.stamp(r) for r in rows)) + except OverflowError as error: + raise TracingPayloadTooLargeError(str(error)) from error + return len(rows) + + # ------------------------------------------------------------ read + + async def list_traces(self, scope: TraceScope, start_ms: int, end_ms: int, cursor: str | None = None) -> TracePage: + return await self.store.list_traces(scope, start_ms, end_ms, cursor) + + async def get_trace(self, trace_id: str, scope: TraceScope, trace_ref: str = "") -> Trace | None: + return await self.store.get_trace(trace_id, scope, trace_ref) + + async def get_span(self, trace_id: str, span_id: str, scope: TraceScope, trace_ref: str = "") -> SpanDetail | None: + return await self.store.get_span(trace_id, span_id, scope, trace_ref) diff --git a/litellm/tracing/store.py b/litellm/tracing/store.py new file mode 100644 index 00000000000..244eddd3def --- /dev/null +++ b/litellm/tracing/store.py @@ -0,0 +1,286 @@ +"""ClickHouse-backed trace store: batched span writes and scoped reads.""" + +import base64 +import binascii +import json +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone +from types import MappingProxyType +from typing import Any, Final + +from litellm.constants import AGENT_TRACING_LIST_PAGE_SIZE +from litellm.integrations.clickhouse.schema import ( + AGENT_TRACES_BY_KEY_TABLE, + OTEL_TRACES_TABLE, +) +from litellm.rust_bridge.traces import TraceStorage +from litellm.tracing.types import ( + AgentNode, + Span, + SpanDetail, + SpanRow, + SpanStatus, + Trace, + TracePage, + TraceScope, + TraceSummary, +) + +NANOS_PER_MS: Final = 1_000_000 +_STATUS: Final = MappingProxyType({"STATUS_CODE_OK": "ok", "STATUS_CODE_ERROR": "error"}) + +_SCOPE_OTEL: Final = ( + "(empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)})" + " AND ({api_key_hash:String} = '' OR ApiKeyHash = {api_key_hash:String})" +) +_TRACE_REF_SQL: Final = "hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))" +LIST_TRACES_SQL: Final = f""" +SELECT TraceId AS trace_id, {_TRACE_REF_SQL} AS trace_ref, + ifNull(any(RootName), '') AS name, any(ServiceName) AS service, + ifNull(any(RootInput), '') AS input_preview, ifNull(any(RootStatus), '') AS status, + toUnixTimestamp64Milli(min(StartTs)) AS start_ms, + dateDiff('millisecond', min(StartTs), max(EndTs)) AS duration_ms, + sum(SpanCount) AS span_count, length(groupUniqArrayArray(AgentNames)) AS agent_count, + sum(AgentCount) AS agent_invocations, + sum(LlmCount) AS llm_calls, sum(ToolCount) AS tool_calls, + sum(InputTokens) AS input_tokens, sum(OutputTokens) AS output_tokens, + groupUniqArrayArray(Models) AS models, sum(ErrorCount) AS error_count +FROM {AGENT_TRACES_BY_KEY_TABLE} +WHERE (empty({{team_ids:Array(String)}}) OR TeamId IN {{team_ids:Array(String)}}) + AND ({{api_key_hash:String}} = '' OR ApiKeyHash = {{api_key_hash:String}}) +GROUP BY TeamId, ApiKeyHash, TraceId +HAVING min(StartTs) >= fromUnixTimestamp64Milli({{start_ms:Int64}}) + AND min(StartTs) < fromUnixTimestamp64Milli({{end_ms:Int64}}) + AND ({{cursor_ms:Int64}} = 0 OR (toUnixTimestamp64Milli(min(StartTs)), trace_ref) + < ({{cursor_ms:Int64}}, {{cursor_trace_id:String}})) +ORDER BY start_ms DESC, trace_ref DESC +LIMIT {{limit:UInt32}} +""" + +TRACE_SPANS_SQL: Final = f""" +SELECT o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, + o.ObservationType AS type, o.AgentName AS agent, o.StatusCode AS status, + o.StatusMessage AS status_message, + toUnixTimestamp64Nano(o.Timestamp) AS start_ns, o.Duration AS duration_ns, + o.ServiceName AS service, o.InputPreview AS input_preview, o.Model AS model, + o.InputTokens AS input_tokens, o.OutputTokens AS output_tokens, + o.LiteLLMRequestId AS litellm_request_id +FROM {OTEL_TRACES_TABLE} AS o +WHERE o.TraceId = {{trace_id:String}} AND {_SCOPE_OTEL} + AND ({{trace_ref:String}} = '' OR {_TRACE_REF_SQL} = {{trace_ref:String}}) +ORDER BY o.Timestamp +LIMIT 1 BY o.SpanId +""" + +SPAN_DETAIL_SQL: Final = f""" +SELECT SpanId AS span_id, Input AS input, Output AS output, SpanAttributes AS attributes +FROM {OTEL_TRACES_TABLE} +WHERE TraceId = {{trace_id:String}} AND SpanId = {{span_id:String}} AND {_SCOPE_OTEL} + AND ({{trace_ref:String}} = '' OR {_TRACE_REF_SQL} = {{trace_ref:String}}) +LIMIT 1 +""" + + +def encode_cursor(start_ms: int, trace_id: str) -> str: + return base64.urlsafe_b64encode(json.dumps((start_ms, trace_id)).encode()).decode() + + +def decode_cursor(cursor: str | None) -> tuple[int, str]: + if not cursor: + return 0, "" + try: + value: Final = json.loads(base64.b64decode(cursor, altchars=b"-_", validate=True)) + if ( + not isinstance(value, list) + or len(value) != 2 + or not isinstance(value[0], int) + or isinstance(value[0], bool) + or value[0] <= 0 + or not isinstance(value[1], str) + or not value[1] + ): + raise ValueError("Invalid trace cursor") + return value[0], value[1] + except (ValueError, UnicodeError, binascii.Error) as error: + raise ValueError("Invalid trace cursor") from error + + +def _iso(ms: int) -> str: + return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).isoformat() + + +def _status(code: str) -> SpanStatus: + return _STATUS.get(code, "unset") + + +def trace_summary_from_row(row: dict[str, Any]) -> TraceSummary: + return TraceSummary( + trace_id=row["trace_id"], + trace_ref=row.get("trace_ref", ""), + name=row["name"], + service=row["service"], + input_preview=row["input_preview"], + start_time=_iso(int(row["start_ms"])), + duration_ms=float(row["duration_ms"]), + status=_status(row["status"]), + span_count=int(row["span_count"]), + agent_count=int(row["agent_count"]), + agent_invocations=int(row.get("agent_invocations") or row["agent_count"]), + llm_calls=int(row["llm_calls"]), + tool_calls=int(row["tool_calls"]), + error_count=int(row.get("error_count") or 0), + input_tokens=int(row["input_tokens"]), + output_tokens=int(row["output_tokens"]), + models=tuple(row["models"]), + ) + + +def span_from_row(row: dict[str, Any], trace_start_ns: int) -> Span: + return Span( + span_id=row["span_id"], + parent_span_id=row["parent_span_id"] or None, + name=row["name"], + type=row["type"], + agent=row["agent"], + start_offset_ms=(int(row["start_ns"]) - trace_start_ns) / NANOS_PER_MS, + duration_ms=int(row["duration_ns"]) / NANOS_PER_MS, + status=_status(row["status"]), + error=row.get("status_message") or None, + input_preview=row["input_preview"], + model=row["model"] or None, + input_tokens=int(row["input_tokens"]), + output_tokens=int(row["output_tokens"]), + litellm_request_id=row["litellm_request_id"] or None, + ) + + +def _parent_agent_of(span: Span, by_id: Mapping[str, Span]) -> str | None: + parent_id = span["parent_span_id"] + for _ in by_id: + if parent_id is None or parent_id not in by_id or parent_id == span["span_id"]: + return None + parent = by_id[parent_id] + if parent["type"] == "agent" and parent["name"] != span["name"]: + return parent["name"] + parent_id = parent["parent_span_id"] + return None + + +def agent_nodes(spans: Sequence[Span]) -> tuple[AgentNode, ...]: + """One node per distinct agent name (200 `researcher` invocations = 1 node), with who invoked it.""" + by_id: Final = MappingProxyType({s["span_id"]: s for s in spans}) + agents: dict[str, AgentNode] = {} # mutable-ok: linear-time aggregation updates counters per agent + for span in spans: + if span["type"] != "agent": + continue + node = agents.setdefault( + span["name"], + AgentNode( + name=span["name"], + parent_agent=_parent_agent_of(span, by_id), + invocations=0, + llm_calls=0, + tool_calls=0, + duration_ms=0.0, + ), + ) + node["invocations"] += 1 + node["duration_ms"] += span["duration_ms"] + for span in spans: + owner = agents.get(span["agent"]) + if owner is None: + continue + if span["type"] == "llm": + owner["llm_calls"] += 1 + elif span["type"] == "tool": + owner["tool_calls"] += 1 + return tuple(agents.values()) + + +def trace_from_rows(trace_id: str, rows: list[dict[str, Any]], trace_ref: str = "") -> Trace | None: + if not rows: + return None + trace_start_ns: Final = min(int(r["start_ns"]) for r in rows) + trace_end_ns: Final = max(int(r["start_ns"]) + int(r["duration_ns"]) for r in rows) + spans: Final = tuple(span_from_row(r, trace_start_ns) for r in rows) + root: Final = next((s for s in spans if s["parent_span_id"] is None), spans[0]) + agents: Final = agent_nodes(spans) + llm_spans: Final = tuple(s for s in spans if s["type"] == "llm") + return Trace( + summary=TraceSummary( + trace_id=trace_id, + trace_ref=trace_ref, + name=root["name"], + service=rows[0]["service"], + input_preview=root["input_preview"], + start_time=_iso(trace_start_ns // NANOS_PER_MS), + duration_ms=(trace_end_ns - trace_start_ns) / NANOS_PER_MS, + status=root["status"], + span_count=len(spans), + agent_count=len(agents), + agent_invocations=sum(a["invocations"] for a in agents), + llm_calls=len(llm_spans), + tool_calls=sum(1 for s in spans if s["type"] == "tool"), + error_count=sum(1 for s in spans if s["status"] == "error"), + input_tokens=sum(s["input_tokens"] for s in spans), + output_tokens=sum(s["output_tokens"] for s in spans), + models=tuple(sorted(frozenset(s["model"] for s in llm_spans if s["model"]))), + ), + agents=agents, + spans=spans, + ) + + +class ClickHouseTraceStore: + """Stores spans and runs scoped trace reads.""" + + def __init__(self, storage: TraceStorage) -> None: + self.storage = storage + + async def insert_spans(self, rows: Sequence[SpanRow]) -> None: + await self.storage.insert_rows(OTEL_TRACES_TABLE, tuple(rows)) + + async def list_traces( + self, + scope: TraceScope, + start_ms: int, + end_ms: int, + cursor: str | None = None, + limit: int = AGENT_TRACING_LIST_PAGE_SIZE, + ) -> TracePage: + cursor_ms, cursor_trace_id = decode_cursor(cursor) + rows = await self.storage.query( + LIST_TRACES_SQL, + MappingProxyType( + { + **scope, + "start_ms": start_ms, + "end_ms": end_ms, + "cursor_ms": cursor_ms, + "cursor_trace_id": cursor_trace_id, + "limit": limit, + } + ), + ) + next_cursor = encode_cursor(int(rows[-1]["start_ms"]), rows[-1]["trace_ref"]) if len(rows) == limit else None + return TracePage(data=tuple(trace_summary_from_row(r) for r in rows), next_cursor=next_cursor) + + async def get_trace(self, trace_id: str, scope: TraceScope, trace_ref: str = "") -> Trace | None: + rows = await self.storage.query( + TRACE_SPANS_SQL, MappingProxyType({**scope, "trace_id": trace_id, "trace_ref": trace_ref}) + ) + return trace_from_rows(trace_id, rows, trace_ref) + + async def get_span(self, trace_id: str, span_id: str, scope: TraceScope, trace_ref: str = "") -> SpanDetail | None: + rows = await self.storage.query( + SPAN_DETAIL_SQL, + MappingProxyType({**scope, "trace_id": trace_id, "span_id": span_id, "trace_ref": trace_ref}), + ) + if not rows: + return None + return SpanDetail( + span_id=rows[0]["span_id"], + input=rows[0]["input"], + output=rows[0]["output"], + attributes=rows[0]["attributes"], + ) diff --git a/litellm/tracing/types.py b/litellm/tracing/types.py new file mode 100644 index 00000000000..b8b6f646111 --- /dev/null +++ b/litellm/tracing/types.py @@ -0,0 +1,120 @@ +""" +Agent tracing types. + +A trace is one agent run. It's made of spans (agent / llm / tool / chain / framework). + Trace + ├── summary: TraceSummary + ├── agents: list[AgentNode] one per distinct agent name (for the agent graph) + └── spans: list[Span] flat, linked by parent_span_id + +""" + +from typing import Literal + +from typing_extensions import NotRequired, ReadOnly, TypedDict + +SpanType = Literal["agent", "llm", "tool", "chain", "framework"] +SpanStatus = Literal["ok", "error", "unset"] + + +class Span(TypedDict): + span_id: ReadOnly[str] + parent_span_id: ReadOnly[str | None] + name: ReadOnly[str] + type: ReadOnly[SpanType] + agent: ReadOnly[str] # the agent this span runs inside, e.g. "researcher" + start_offset_ms: ReadOnly[float] # relative to trace start + duration_ms: ReadOnly[float] + status: ReadOnly[SpanStatus] + error: ReadOnly[str | None] # exception message when status == "error" + input_preview: ReadOnly[str] + model: ReadOnly[str | None] + input_tokens: ReadOnly[int] + output_tokens: ReadOnly[int] + litellm_request_id: ReadOnly[str | None] + + +class AgentNode(TypedDict): + """One distinct agent in a trace. 200 invocations of `researcher` = one node.""" + + name: ReadOnly[str] + parent_agent: ReadOnly[str | None] + invocations: int + llm_calls: int + tool_calls: int + duration_ms: float + + +class TraceSummary(TypedDict): + trace_id: ReadOnly[str] + trace_ref: ReadOnly[NotRequired[str]] + name: ReadOnly[str] + service: ReadOnly[str] + input_preview: ReadOnly[str] + start_time: ReadOnly[str] # ISO 8601 + duration_ms: ReadOnly[float] + status: ReadOnly[SpanStatus] + span_count: ReadOnly[int] + agent_count: ReadOnly[int] # distinct agent names (researcher x200 counts once) + agent_invocations: ReadOnly[int] # agent spans (researcher x200 counts 200) + llm_calls: ReadOnly[int] + tool_calls: ReadOnly[int] + error_count: ReadOnly[int] # spans with an error status; > 0 means the run shows as failed + input_tokens: ReadOnly[int] + output_tokens: ReadOnly[int] + models: ReadOnly[tuple[str, ...]] + + +class Trace(TypedDict): + summary: ReadOnly[TraceSummary] + agents: ReadOnly[tuple[AgentNode, ...]] + spans: ReadOnly[tuple[Span, ...]] + + +class TracePage(TypedDict): + data: ReadOnly[tuple[TraceSummary, ...]] + next_cursor: ReadOnly[str | None] + + +class SpanDetail(TypedDict): + span_id: ReadOnly[str] + input: ReadOnly[str] + output: ReadOnly[str] + attributes: ReadOnly[dict[str, str]] + + +class TraceScope(TypedDict): + """Who is asking. Empty team_ids = all teams (admins only).""" + + team_ids: ReadOnly[tuple[str, ...]] + api_key_hash: ReadOnly[str] + + +class SpanRow(TypedDict): + """One stored span (ClickHouse `otel_traces` row). Produced by `litellm.tracing.decode`.""" + + Timestamp: ReadOnly[int] # unix ns + TraceId: ReadOnly[str] + SpanId: ReadOnly[str] + ParentSpanId: ReadOnly[str] + TraceState: ReadOnly[str] + SpanName: ReadOnly[str] + SpanKind: ReadOnly[str] + ServiceName: ReadOnly[str] + ResourceAttributes: dict[str, str] + ScopeName: ReadOnly[str] + ScopeVersion: ReadOnly[str] + SpanAttributes: dict[str, str] + Duration: ReadOnly[int] # ns + StatusCode: ReadOnly[str] + StatusMessage: ReadOnly[str] + TeamId: str + ApiKeyHash: str + ObservationType: SpanType + AgentName: str + LiteLLMRequestId: str + Model: str + InputTokens: int + OutputTokens: int + Input: str + Output: str diff --git a/tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py b/tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py new file mode 100644 index 00000000000..e5b00b3c783 --- /dev/null +++ b/tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py @@ -0,0 +1,67 @@ +""" +Tests for the CustomBatchLogger-based ClickHouse base logger. +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.integrations.clickhouse import clickhouse_batch_logger as module +from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger +from litellm.integrations.custom_batch_logger import CustomBatchLogger + + +class _TestLogger(ClickHouseBatchLogger): + table = "test_table" + + +def _logger(insert: AsyncMock) -> _TestLogger: + storage = MagicMock() + storage.insert_rows = insert + return _TestLogger(storage=storage) + + +def test_is_a_custom_batch_logger(): + assert issubclass(ClickHouseBatchLogger, CustomBatchLogger) + + +@pytest.mark.asyncio +async def test_flush_splits_into_batches_and_empties_queue(): + insert = AsyncMock() + logger = _logger(insert) + logger.batch_size = 2 + logger.log_queue.extend([{"i": i} for i in range(5)]) + + await logger.flush_queue() + + assert [len(c.args[1]) for c in insert.await_args_list] == [2, 2, 1] + assert all(c.args[0] == "test_table" for c in insert.await_args_list) + assert logger.log_queue == [] + assert logger.rows_written == 5 + + +@pytest.mark.asyncio +async def test_is_full_signals_backpressure(): + logger = _logger(AsyncMock()) + with patch.object(module, "CLICKHOUSE_MAX_BUFFERED_ROWS", 3): + logger.log_queue.extend([{}, {}]) + assert logger.is_full() is False + logger.log_queue.append({}) + assert logger.is_full() is True + + +@pytest.mark.asyncio +async def test_failed_insert_is_requeued_then_dropped(): + insert = AsyncMock(side_effect=RuntimeError("clickhouse down")) + logger = _logger(insert) + logger.log_queue.extend([{"request_id": "a"}, {"request_id": "b"}]) + + with patch.object(module, "CLICKHOUSE_MAX_RETRIES", 2): + await logger.flush_queue() + assert len(logger.log_queue) == 2 # kept for retry + await logger.flush_queue() + + assert insert.await_count == 2 + assert logger.rows_dropped == 2 + assert logger.rows_written == 0 + assert logger.log_queue == [] diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index 6087aa1f570..e3851f6c21a 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -1,3 +1,4 @@ +import gzip import io import json from collections.abc import Mapping @@ -31,12 +32,14 @@ from litellm.proxy.common_utils.http_parsing_utils import ( ) -def _starlette_request(body: bytes, content_type: str) -> Request: +def _starlette_request( + body: bytes, content_type: str, path: str = "/v1/messages", content_encoding: str = "" +) -> Request: scope = { "type": "http", "method": "POST", - "path": "/v1/messages", - "headers": [(b"content-type", content_type.encode())], + "path": path, + "headers": [(b"content-type", content_type.encode()), (b"content-encoding", content_encoding.encode())], "query_string": b"", } chunks = iter((body,)) @@ -72,6 +75,26 @@ async def test_read_raw_json_body_is_none_for_form_bodies(): assert await read_raw_json_body(request) is None +@pytest.mark.asyncio +@pytest.mark.parametrize("content_type", ["application/x-protobuf", "application/protobuf; charset=binary"]) +async def test_protobuf_body_is_not_parsed_as_json(content_type): + # OTLP trace exports (POST /v1/traces) are binary protobuf; arbitrary bytes like these + # used to hit the JSON surrogate-repair path and fail auth with a 400. + body = b"\n\xa2\x01\n\x1c\n\x0cservice.name\x12\x0c\n\nswarm\xed\xa0\x80\xff" + request = _starlette_request(body, content_type) + + assert await _read_request_body(request) == {} + assert await request.body() == body # body is still readable by the endpoint + + +@pytest.mark.asyncio +async def test_gzipped_json_trace_body_survives_auth_pre_read(): + body = gzip.compress(b'{"resourceSpans": []}') + request = _starlette_request(body, "application/json", "/v1/traces", "gzip") + assert await _read_request_body(request) == {} + assert await request.body() == body + + @pytest.mark.asyncio async def test_read_raw_json_body_is_none_for_a_request_that_only_mocks_the_parsed_body_path(): mock_request = MagicMock() diff --git a/tests/test_litellm/proxy/test_tracing_endpoints.py b/tests/test_litellm/proxy/test_tracing_endpoints.py new file mode 100644 index 00000000000..4c7c70a39f3 --- /dev/null +++ b/tests/test_litellm/proxy/test_tracing_endpoints.py @@ -0,0 +1,180 @@ +""" +Tests for the agent tracing endpoints (litellm/proxy/tracing_endpoints.py). +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient + +from litellm.proxy import tracing_endpoints +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.tracing import TracingPayloadTooLargeError + +TEAM_KEY = UserAPIKeyAuth( + token="hashed-key", team_id="team-research", org_id="org-1", user_role=LitellmUserRoles.INTERNAL_USER +) + + +# ---------------------------------------------------------------- scope / tenant + + +def test_scope_for_admin_sees_everything(): + for role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + auth = UserAPIKeyAuth(token="k", team_id="team-a", user_role=role) + assert tracing_endpoints.scope_for(auth) == {"team_ids": (), "api_key_hash": ""} + + +def test_scope_for_team_key_sees_its_team(): + assert tracing_endpoints.scope_for(TEAM_KEY) == {"team_ids": ("team-research",), "api_key_hash": ""} + + +def test_scope_for_teamless_key_sees_only_its_own_traces(): + auth = UserAPIKeyAuth(token="hashed-key", user_role=LitellmUserRoles.INTERNAL_USER) + assert tracing_endpoints.scope_for(auth) == {"team_ids": ("",), "api_key_hash": "hashed-key"} + + +def test_scope_for_no_team_no_token_is_forbidden(): + with pytest.raises(HTTPException) as e: + tracing_endpoints.scope_for(UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER)) + assert e.value.status_code == 403 + + +def test_tenant_for_comes_from_auth(): + tenant = tracing_endpoints.tenant_for(TEAM_KEY) + assert (tenant.team_id, tenant.api_key_hash, tenant.org_id) == ("team-research", "hashed-key", "org-1") + blank = tracing_endpoints.tenant_for(UserAPIKeyAuth()) + assert (blank.team_id, blank.api_key_hash, blank.org_id) == ("", "", "") + + +# ---------------------------------------------------------------- endpoints + + +@pytest.fixture +def receiver(monkeypatch) -> MagicMock: + fake = MagicMock() + fake.ingest = AsyncMock(return_value=1) + fake.list_traces = AsyncMock(return_value={"data": [], "next_cursor": None}) + fake.get_trace = AsyncMock(return_value=None) + fake.get_span = AsyncMock(return_value=None) + monkeypatch.setattr(tracing_endpoints, "receiver", fake) + return fake + + +@pytest.fixture +def client() -> TestClient: + app = FastAPI() + app.include_router(tracing_endpoints.router) + app.dependency_overrides[user_api_key_auth] = lambda: TEAM_KEY + return TestClient(app) + + +def test_501_when_tracing_not_enabled(client, monkeypatch): + monkeypatch.setattr(tracing_endpoints, "receiver", None) + assert client.post("/v1/traces", content=b"").status_code == 501 + assert client.get("/v1/traces").status_code == 501 + + +def test_post_protobuf_returns_empty_protobuf(client, receiver): + response = client.post( + "/v1/traces", + content=b"\x0a\x00", + headers={"content-type": "application/x-protobuf", "content-encoding": "gzip"}, + ) + assert response.status_code == 200 + assert response.content == b"" + assert response.headers["content-type"] == "application/x-protobuf" + kwargs = receiver.ingest.call_args.kwargs + assert kwargs["body"] == b"\x0a\x00" + assert kwargs["content_type"] == "application/x-protobuf" + assert kwargs["content_encoding"] == "gzip" + assert kwargs["tenant"].team_id == "team-research" + + +def test_post_json_returns_empty_json(client, receiver): + response = client.post("/v1/traces", content=b"{}", headers={"content-type": "application/json"}) + assert response.status_code == 200 + assert response.json() == {} + + +def test_post_clickhouse_failure_is_503_with_retry_after(client, receiver): + receiver.ingest.side_effect = RuntimeError("ClickHouse unavailable") + response = client.post("/v1/traces", content=b"", headers={"content-type": "application/x-protobuf"}) + assert response.status_code == 503 + assert response.headers["retry-after"] == str(tracing_endpoints.OTLP_RETRY_AFTER_SECONDS) + + +def test_post_too_large_is_413(client, receiver): + receiver.ingest.side_effect = TracingPayloadTooLargeError("OTLP body exceeds 10 bytes") + response = client.post("/v1/traces", content=b"x" * 20) + assert response.status_code == 413 + assert "exceeds" in response.json()["detail"] + + +def test_list_traces_passes_scope_window_and_cursor(client, receiver): + response = client.get("/v1/traces", params={"start_ms": 1, "end_ms": 2, "cursor": "abc"}) + assert response.status_code == 200 + assert response.json() == {"data": [], "next_cursor": None} + receiver.list_traces.assert_awaited_once_with( + scope={"team_ids": ("team-research",), "api_key_hash": ""}, start_ms=1, end_ms=2, cursor="abc" + ) + + +def test_list_traces_defaults_to_last_24h(client, receiver): + client.get("/v1/traces") + kwargs = receiver.list_traces.call_args.kwargs + assert kwargs["end_ms"] - kwargs["start_ms"] == tracing_endpoints.MS_PER_DAY + assert kwargs["cursor"] is None + + +def test_get_trace_404_and_200(client, receiver): + assert client.get("/v1/traces/missing").status_code == 404 + trace = {"summary": {"trace_id": "t1"}, "agents": [], "spans": []} + receiver.get_trace.return_value = trace + response = client.get("/v1/traces/t1") + assert response.status_code == 200 + assert response.json() == trace + receiver.get_trace.assert_awaited_with("t1", {"team_ids": ("team-research",), "api_key_hash": ""}, "") + + +def test_get_span_404_and_200(client, receiver): + assert client.get("/v1/traces/t1/spans/s1").status_code == 404 + receiver.get_span.return_value = {"span_id": "s1", "input": "", "output": "", "attributes": {}} + response = client.get("/v1/traces/t1/spans/s1") + assert response.status_code == 200 + assert response.json()["span_id"] == "s1" + receiver.get_span.assert_awaited_with("t1", "s1", {"team_ids": ("team-research",), "api_key_hash": ""}, "") + + +def test_trace_detail_passes_scoped_reference(client, receiver): + receiver.get_trace.return_value = {"summary": {"trace_id": "t1"}, "agents": [], "spans": []} + assert client.get("/v1/traces/t1?trace_ref=run-one").status_code == 200 + receiver.get_trace.assert_awaited_with("t1", {"team_ids": ("team-research",), "api_key_hash": ""}, "run-one") + + +def test_invalid_export_and_cursor_are_client_errors(client, receiver): + from litellm.tracing.decode import InvalidOTLPPayloadError + + receiver.ingest.side_effect = InvalidOTLPPayloadError("invalid OTLP trace payload") + assert client.post("/v1/traces", content=b"broken").status_code == 400 + receiver.list_traces.side_effect = ValueError("Invalid trace cursor") + assert client.get("/v1/traces?cursor=broken").status_code == 400 + + +def test_teamless_key_without_token_gets_403_on_reads(client, receiver): + client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER + ) + assert client.get("/v1/traces").status_code == 403 + receiver.list_traces.assert_not_called() + + +def test_view_only_admin_cannot_ingest_traces(client, receiver): + client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + token="admin-key", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ) + response = client.post("/v1/traces", content=b"{}") + assert response.status_code == 403 + receiver.ingest.assert_not_called() diff --git a/tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json b/tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json new file mode 100644 index 00000000000..9bd8e67633b --- /dev/null +++ b/tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json @@ -0,0 +1,924 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "86db1687-77ed-422d-a6f7-0319594d9158" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "agent-demo" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "langsmith" + }, + "spans": [ + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "XnnztbUEmF4=", + "name": "deep_research_agent", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742989377137920", + "endTimeUnixNano": "1790743040762587136", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "deepagents" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJtZXNzYWdlcyI6W3siY29udGVudCI6IlNob3VsZCB3ZSBzdG9yZSBPVEVMIGFnZW50IHNwYW5zIGluIENsaWNrSG91c2Ugb3IgUG9zdGdyZXMgYXQgNTBrIHNwYW5zL3NlYz8iLCJhZGRpdGlvbmFsX2t3YXJncyI6e30sInJlc3BvbnNlX21ldGFkYXRhIjp7fSwidHlwZSI6Imh1bWFuIiwiaWQiOiJiMTljODgzMS0wOWIwLTQ5ZjYtYjdlYS05YzQ3ZTM4OWNjMDAifV19" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJtZXNzYWdlcyI6W3siY29udGVudCI6IlNob3VsZCB3ZSBzdG9yZSBPVEVMIGFnZW50IHNwYW5zIGluIENsaWNrSG91c2Ugb3IgUG9zdGdyZXMgYXQgNTBrIHNwYW5zL3NlYz8iLCJhZGRpdGlvbmFsX2t3YXJncyI6e30sInJlc3BvbnNlX21ldGFkYXRhIjp7fSwidHlwZSI6Imh1bWFuIiwiaWQiOiJiMTljODgzMS0wOWIwLTQ5ZjYtYjdlYS05YzQ3ZTM4OWNjMDAifSx7ImNvbnRlbnQiOiJJJ2xsIGhlbHAgeW91IGRlY2lkZSBiZXR3ZWVuIENsaWNrSG91c2UgYW5kIFBvc3RncmVzIGZvciBzdG9yaW5nIE9wZW5UZWxlbWV0cnkgc3BhbnMgYXQgNTBrIHNwYW5zL3NlYy4gTGV0IG1lIHJlc2VhcmNoIHRoaXMgc3lzdGVtYXRpY2FsbHkuIiwiYWRkaXRpb25hbF9rd2FyZ3MiOnsicmVmdXNhbCI6bnVsbH0sInJlc3BvbnNlX21ldGFkYXRhIjp7InRva2VuX3VzYWdlIjp7ImNvbXBsZXRpb25fdG9rZW5zIjo0NjcsInByb21wdF90b2tlbnMiOjMzMzIsInRvdGFsX3Rva2VucyI6Mzc5OSwiY29tcGxldGlvbl90b2tlbnNfZGV0YWlscyI6eyJhY2NlcHRlZF9wcmVkaWN0aW9uX3Rva2VucyI6bnVsbCwiYXVkaW9fdG9rZW5zIjpudWxsLCJyZWFzb25pbmdfdG9rZW5zIjowLCJyZWplY3RlZF9wcmVkaWN0aW9uX3Rva2VucyI6bnVsbCwidGV4dF90b2tlbnMiOjQ2N30sInByb21wdF90b2tlbnNfZGV0YWlscyI6eyJhdWRpb190b2tlbnMiOm51bGwsImNhY2hlX3dyaXRlX3Rva2VucyI6MzMyOSwiY2FjaGVkX3Rva2VucyI6MCwiaW1hZ2VfdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6MywiY2FjaGVfY3JlYXRpb25fdG9rZW5zIjozMzI5LCJjYWNoZV9jcmVhdGlvbl90b2tlbl9kZXRhaWxzIjp7ImVwaGVtZXJhbF81bV9pbnB1dF90b2tlbnMiOjMzMjksImVwaGVtZXJhbF8xaF9pbnB1dF90b2tlbnMiOjB9fSwiY2FjaGVfY3JlYXRpb25faW5wdXRfdG9rZW5zIjozMzI5LCJjYWNoZV9yZWFkX2lucHV0X3Rva2VucyI6MCwiaW5mZXJlbmNlX2dlbyI6Im5vdF9hdmFpbGFibGUiLCJzZXJ2aWNlX3RpZXIiOiJzdGFuZGFyZCJ9LCJtb2RlbF9wcm92aWRlciI6Im9wZW5haSIsIm1vZGVsX25hbWUiOiJjbGF1ZGUtc29ubmV0LTQtNSIsInN5c3RlbV9maW5nZXJwcmludCI6bnVsbCwiaWQiOiJjaGF0Y21wbC00MDc3YmIzNi05MzgwLTRhM2ItOTQ4MS0yNDU3MDBjZWYwOWEiLCJmaW5pc2hfcmVhc29uIjoidG9vbF9jYWxscyIsImxvZ3Byb2JzIjpudWxsfSwidHlwZSI6ImFpIiwibmFtZSI6ImRlZXBfcmVzZWFyY2hfYWdlbnQiLCJpZCI6ImxjX3J1bi0tMDFhMGYwOTktOGE0Ny03ZTQyLWE1ZjQtNWM0N2RlM2QxY2VjLTAiLCJ0b29sX2NhbGxzIjpbeyJuYW1lIjoid3JpdGVfZmlsZSIsImFyZ3MiOnsiZmlsZV9wYXRoIjoiL3RtcC9yZXNlYXJjaF90b2Rvcy5tZCIsImNvbnRlbnQiOiIjIFJlc2VhcmNoIFBsYW46IENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIE9URUwgU3BhbnMgKDUway9zZWMpXG5cbiMjIFRhc2tzXG4tIFsgXSBSZXNlYXJjaCBDbGlja0hvdXNlIGFuZCBQb3N0Z3JlcyBjYXBhYmlsaXRpZXMgZm9yIGhpZ2gtdm9sdW1lIHRpbWUtc2VyaWVzIGRhdGFcbi4uLiJ9LCJpZCI6InRvb2x1XzAxNjFYaFlQM0I1Zmc0VTFwc1QzcGNpUiIsInR5cGUiOiJ0b29sX2NhbGwifSx7Im5hbWUiOiJ0YXNrIiwiYXJncyI6eyJzdWJhZ2VudF90eXBlIjoicmVzZWFyY2hlciIsImRlc2NyaXB0aW9uIjoiUmVzZWFyY2ggYW5kIGNvbXBhcmUgQ2xpY2tIb3VzZSB2cyBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IChPVEVMKSBhZ2VudCBzcGFucyBhdCA1MCwwMDAgc3BhbnMgcGVyIHNlY29uZC5cblxuRm9jdXMgb246XG4xLiBXcml0ZSB0aHJvdWdocHV0IGNhcGFiaWxpdGllcy4uLiJ9LCJpZCI6InRvb2x1XzAxUEx5bzhUS0tUcFhSNGZwOTZEbjkzVyIsInR5cGUiOiJ0b29sX2NhbGwifV0sImludmFsaWRfdG9vbF9jYWxscyI6W10sInVzYWdlX21ldGFkYXRhIjp7ImlucHV0X3Rva2VucyI6MzMzMiwib3V0cHV0X3Rva2VucyI6NDY3LCJ0b3RhbF90b2tlbnMiOjM3OTksImlucHV0X3Rva2VuX2RldGFpbHMiOnsiY2FjaGVfcmVhZCI6MCwiY2FjaGVfY3JlYXRpb24iOjMzMjl9LCJvdXRwdXRfdG9rZW5fZGV0YWlscyI6eyJyZWFzb25pbmciOjB9fX0seyJjb250ZW50IjoiQmFzZWQgb24gbXkgcmVzZWFyY2gsIGhlcmUncyBhIGNvbXByZWhlbnNpdmUgY29tcGFyaXNvbiBvZiAqKkNsaWNrSG91c2UgdnMgUG9zdGdyZXMqKiBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IHNwYW5zIGF0IDUwLDAwMCBzcGFucy9zZWNvbmQ6XG5cbiMjICoqMS4gV3JpdGUgVGhyLi4uIiwiYWRkaXRpb25hbF9rd2FyZ3MiOnt9LCJyZXNwb25zZV9tZXRhZGF0YSI6e30sInR5cGUiOiJ0b29sIiwibmFtZSI6InRhc2siLCJpZCI6IjE0MzVkZTNjLWI4NzktNDQ2YS04MDU0LTFiMGI4MjQ1YmZhZSIsInRvb2xfY2FsbF9pZCI6InRvb2x1XzAxUEx5bzhUS0tUcFhSNGZwOTZEbjkzVyIsInN0YXR1cyI6InN1Y2Nlc3MifSx7ImNvbnRlbnQiOiJCYXNlZCBvbiB0aGUgcmVzZWFyY2ggZmluZGluZ3MsIGhlcmUncyBteSByZWNvbW1lbmRhdGlvbjpcblxuIyMgUmVjb21tZW5kYXRpb246ICoqVXNlIENsaWNrSG91c2UqKlxuXG4qKkNsaWNrSG91c2UgaXMgdGhlIGNsZWFyIGNob2ljZSoqIGZvciBzdG9yaW5nIDUwayBPVEVMIHNwYW5zLy4uLiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7InJlZnVzYWwiOm51bGx9LCJyZXNwb25zZV9tZXRhZGF0YSI6eyJ0b2tlbl91c2FnZSI6eyJjb21wbGV0aW9uX3Rva2VucyI6MjQxLCJwcm9tcHRfdG9rZW5zIjo0NTcwLCJ0b3RhbF90b2tlbnMiOjQ4MTEsImNvbXBsZXRpb25fdG9rZW5zX2RldGFpbHMiOnsiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsImF1ZGlvX3Rva2VucyI6bnVsbCwicmVhc29uaW5nX3Rva2VucyI6MCwicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjoyNDF9LCJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOnsiYXVkaW9fdG9rZW5zIjpudWxsLCJjYWNoZV93cml0ZV90b2tlbnMiOjEyMzQsImNhY2hlZF90b2tlbnMiOjMzMjksImltYWdlX3Rva2VucyI6bnVsbCwidGV4dF90b2tlbnMiOjcsImNhY2hlX2NyZWF0aW9uX3Rva2VucyI6MTIzNCwiY2FjaGVfY3JlYXRpb25fdG9rZW5fZGV0YWlscyI6eyJlcGhlbWVyYWxfNW1faW5wdXRfdG9rZW5zIjoxMjM0LCJlcGhlbWVyYWxfMWhfaW5wdXRfdG9rZW5zIjowfX0sImNhY2hlX2NyZWF0aW9uX2lucHV0X3Rva2VucyI6MTIzNCwiY2FjaGVfcmVhZF9pbnB1dF90b2tlbnMiOjMzMjksImluZmVyZW5jZV9nZW8iOiJub3RfYXZhaWxhYmxlIiwic2VydmljZV90aWVyIjoic3RhbmRhcmQifSwibW9kZWxfcHJvdmlkZXIiOiJvcGVuYWkiLCJtb2RlbF9uYW1lIjoiY2xhdWRlLXNvbm5ldC00LTUiLCJzeXN0ZW1fZmluZ2VycHJpbnQiOm51bGwsImlkIjoiY2hhdGNtcGwtZjI2Y2NiNDUtYWIxYi00NGM2LWJkOWUtNDFhMDJjYTVmMTRkIiwiZmluaXNoX3JlYXNvbiI6InN0b3AiLCJsb2dwcm9icyI6bnVsbH0sInR5cGUiOiJhaSIsIm5hbWUiOiJkZWVwX3Jlc2VhcmNoX2FnZW50IiwiaWQiOiJsY19ydW4tLTAxYTBmMDlhLTM4ZTEtNzc0My04ZGIwLTNjNjU5YjdlMGY2MC0wIiwidG9vbF9jYWxscyI6W10sImludmFsaWRfdG9vbF9jYWxscyI6W10sInVzYWdlX21ldGFkYXRhIjp7ImlucHV0X3Rva2VucyI6NDU3MCwib3V0cHV0X3Rva2VucyI6MjQxLCJ0b3RhbF90b2tlbnMiOjQ4MTEsImlucHV0X3Rva2VuX2RldGFpbHMiOnsiY2FjaGVfcmVhZCI6MzMyOSwiY2FjaGVfY3JlYXRpb24iOjEyMzR9LCJvdXRwdXRfdG9rZW5fZGV0YWlscyI6eyJyZWFzb25pbmciOjB9fX1dLCJmaWxlcyI6eyIvdG1wL3Jlc2VhcmNoX3RvZG9zLm1kIjp7ImNvbnRlbnQiOiIjIFJlc2VhcmNoIFBsYW46IENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIE9URUwgU3BhbnMgKDUway9zZWMpXG5cbiMjIFRhc2tzXG4tIFsgXSBSZXNlYXJjaCBDbGlja0hvdXNlIGFuZCBQb3N0Z3JlcyBjYXBhYmlsaXRpZXMgZm9yIGhpZ2gtdm9sdW1lIHRpbWUtc2VyaWVzIGRhdGFcbi4uLiIsImVuY29kaW5nIjoidXRmLTgiLCJjcmVhdGVkX2F0IjoiMjAyNi0wOS0zMFQwNDozNjozOC44OTkwNTArMDA6MDAiLCJtb2RpZmllZF9hdCI6IjIwMjYtMDktMzBUMDQ6MzY6MzguODk5MDUwKzAwOjAwIn19fQ==" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "imocMZQNB68=", + "parentSpanId": "Hfr3D90RhPI=", + "name": "ChatOpenAI", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742989383207936", + "endTimeUnixNano": "1790742998893985024", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chat" + } + }, + { + "key": "gen_ai.serialized.name", + "value": { + "stringValue": "ChatOpenAI" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "llm" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "ChatOpenAI" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "anthropic" + } + }, + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "claude-sonnet-4-5" + } + }, + { + "key": "gen_ai.tool.definitions", + "value": { + "stringValue": "[{\"type\":\"function\",\"function\":{\"name\":\"ls\",\"description\":\"Lists all files in a directory.\\n\\nThis is useful for exploring the filesystem and finding the right file to read or edit.\\nYou should almost ALWAYS use this tool before using the read_file or edit_file tools.\",\"parameters\":{\"properties\":{\"path\":{\"description\":\"Absolute path to the directory to list. Must be absolute, not relative.\",\"type\":\"string\"}},\"required\":[\"path\"],\"type\":\"object\"}}}]" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "langchain_chat_model" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\",\"langchain-core\":\"1.6.6\",\"langchain\":\"1.4.3\",\"langchain-openai\":\"1.6.6\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "2" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "model" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"branch:to:model\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_pull\",\"model\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "model:9abb6d12-32f9-4289-15b6-36ac41ba926c" + } + }, + { + "key": "langsmith.metadata.checkpoint_ns", + "value": { + "stringValue": "model:9abb6d12-32f9-4289-15b6-36ac41ba926c" + } + }, + { + "key": "langsmith.metadata.ls_provider", + "value": { + "stringValue": "openai" + } + }, + { + "key": "langsmith.metadata.ls_model_name", + "value": { + "stringValue": "claude-sonnet-4-5" + } + }, + { + "key": "langsmith.metadata.ls_model_type", + "value": { + "stringValue": "chat" + } + }, + { + "key": "langsmith.metadata.ls_max_tokens", + "value": { + "intValue": "700" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.model", + "value": { + "stringValue": "claude-sonnet-4-5" + } + }, + { + "key": "langsmith.metadata.model_name", + "value": { + "stringValue": "claude-sonnet-4-5" + } + }, + { + "key": "langsmith.metadata.stream", + "value": { + "boolValue": false + } + }, + { + "key": "langsmith.metadata.max_completion_tokens", + "value": { + "intValue": "700" + } + }, + { + "key": "langsmith.metadata._type", + "value": { + "stringValue": "openai-chat" + } + }, + { + "key": "langsmith.metadata.usage_metadata", + "value": { + "stringValue": "{\"input_tokens\":3332,\"output_tokens\":467,\"total_tokens\":3799,\"input_token_details\":{\"cache_read\":0,\"cache_creation\":3329},\"output_token_details\":{\"reasoning\":0}}" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "langsmith.span.tags", + "value": { + "stringValue": "seq:step:1" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJtZXNzYWdlcyI6W1t7ImxjIjoxLCJ0eXBlIjoiY29uc3RydWN0b3IiLCJpZCI6WyJsYW5nY2hhaW4iLCJzY2hlbWEiLCJtZXNzYWdlcyIsIlN5c3RlbU1lc3NhZ2UiXSwia3dhcmdzIjp7ImNvbnRlbnQiOiJZb3UgYXJlIGEgcmVzZWFyY2ggbGVhZC4gUGxhbiB3aXRoIHdyaXRlX3RvZG9zLCBkZWxlZ2F0ZSBvbmUgcXVlc3Rpb24gdG8gdGhlIHJlc2VhcmNoZXIgc3ViYWdlbnQgdmlhIHRhc2ssIHRoZW4gd3JpdGUgYSBzaG9ydCByZWNvbW1lbmRhdGlvbiAoPD01IHNlbnRlbmNlcykuIiwidHlwZSI6InN5c3RlbSJ9fSx7ImxjIjoxLCJ0eXBlIjoiY29uc3RydWN0b3IiLCJpZCI6WyJsYW5nY2hhaW4iLCJzY2hlbWEiLCJtZXNzYWdlcyIsIkh1bWFuTWVzc2FnZSJdLCJrd2FyZ3MiOnsiY29udGVudCI6IlNob3VsZCB3ZSBzdG9yZSBPVEVMIGFnZW50IHNwYW5zIGluIENsaWNrSG91c2Ugb3IgUG9zdGdyZXMgYXQgNTBrIHNwYW5zL3NlYz8iLCJ0eXBlIjoiaHVtYW4iLCJpZCI6ImIxOWM4ODMxLTA5YjAtNDlmNi1iN2VhLTljNDdlMzg5Y2MwMCJ9fV1dfQ==" + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "3332" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "467" + } + }, + { + "key": "gen_ai.usage.total_tokens", + "value": { + "intValue": "3799" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJnZW5lcmF0aW9ucyI6W1t7InRleHQiOiJJJ2xsIGhlbHAgeW91IGRlY2lkZSBiZXR3ZWVuIENsaWNrSG91c2UgYW5kIFBvc3RncmVzIGZvciBzdG9yaW5nIE9wZW5UZWxlbWV0cnkgc3BhbnMgYXQgNTBrIHNwYW5zL3NlYy4gTGV0IG1lIHJlc2VhcmNoIHRoaXMgc3lzdGVtYXRpY2FsbHkuIiwiZ2VuZXJhdGlvbl9pbmZvIjp7ImZpbmlzaF9yZWFzb24iOiJ0b29sX2NhbGxzIiwibG9ncHJvYnMiOm51bGx9LCJ0eXBlIjoiQ2hhdEdlbmVyYXRpb24iLCJtZXNzYWdlIjp7ImxjIjoxLCJ0eXBlIjoiY29uc3RydWN0b3IiLCJpZCI6WyJsYW5nY2hhaW4iLCJzY2hlbWEiLCJtZXNzYWdlcyIsIkFJTWVzc2FnZSJdLCJrd2FyZ3MiOnsiY29udGVudCI6IkknbGwgaGVscCB5b3UgZGVjaWRlIGJldHdlZW4gQ2xpY2tIb3VzZSBhbmQgUG9zdGdyZXMgZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSBzcGFucyBhdCA1MGsgc3BhbnMvc2VjLiBMZXQgbWUgcmVzZWFyY2ggdGhpcyBzeXN0ZW1hdGljYWxseS4iLCJhZGRpdGlvbmFsX2t3YXJncyI6eyJyZWZ1c2FsIjpudWxsfSwicmVzcG9uc2VfbWV0YWRhdGEiOnsidG9rZW5fdXNhZ2UiOnsiY29tcGxldGlvbl90b2tlbnMiOjQ2NywicHJvbXB0X3Rva2VucyI6MzMzMiwidG90YWxfdG9rZW5zIjozNzk5LCJjb21wbGV0aW9uX3Rva2Vuc19kZXRhaWxzIjp7ImFjY2VwdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjpudWxsLCJhdWRpb190b2tlbnMiOm51bGwsInJlYXNvbmluZ190b2tlbnMiOjAsInJlamVjdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6NDY3fSwicHJvbXB0X3Rva2Vuc19kZXRhaWxzIjp7ImF1ZGlvX3Rva2VucyI6bnVsbCwiY2FjaGVfd3JpdGVfdG9rZW5zIjozMzI5LCJjYWNoZWRfdG9rZW5zIjowLCJpbWFnZV90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjozLCJjYWNoZV9jcmVhdGlvbl90b2tlbnMiOjMzMjksImNhY2hlX2NyZWF0aW9uX3Rva2VuX2RldGFpbHMiOnsiZXBoZW1lcmFsXzVtX2lucHV0X3Rva2VucyI6MzMyOSwiZXBoZW1lcmFsXzFoX2lucHV0X3Rva2VucyI6MH19LCJjYWNoZV9jcmVhdGlvbl9pbnB1dF90b2tlbnMiOjMzMjksImNhY2hlX3JlYWRfaW5wdXRfdG9rZW5zIjowLCJpbmZlcmVuY2VfZ2VvIjoibm90X2F2YWlsYWJsZSIsInNlcnZpY2VfdGllciI6InN0YW5kYXJkIn0sIm1vZGVsX3Byb3ZpZGVyIjoib3BlbmFpIiwibW9kZWxfbmFtZSI6ImNsYXVkZS1zb25uZXQtNC01Iiwic3lzdGVtX2ZpbmdlcnByaW50IjpudWxsLCJpZCI6ImNoYXRjbXBsLTQwNzdiYjM2LTkzODAtNGEzYi05NDgxLTI0NTcwMGNlZjA5YSIsImZpbmlzaF9yZWFzb24iOiJ0b29sX2NhbGxzIiwibG9ncHJvYnMiOm51bGx9LCJ0eXBlIjoiYWkiLCJpZCI6ImxjX3J1bi0tMDFhMGYwOTktOGE0Ny03ZTQyLWE1ZjQtNWM0N2RlM2QxY2VjLTAiLCJ0b29sX2NhbGxzIjpbeyJuYW1lIjoid3JpdGVfZmlsZSIsImFyZ3MiOnsiZmlsZV9wYXRoIjoiL3RtcC9yZXNlYXJjaF90b2Rvcy5tZCIsImNvbnRlbnQiOiIjIFJlc2VhcmNoIFBsYW46IENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIE9URUwgU3BhbnMgKDUway9zZWMpXG5cbiMjIFRhc2tzXG4tIFsgXSBSZXNlYXJjaCBDbGlja0hvdXNlIGFuZCBQb3N0Z3JlcyBjYXBhYmlsaXRpZXMgZm9yIGhpZ2gtdm9sdW1lIHRpbWUtc2VyaWVzIGRhdGFcbi4uLiJ9LCJpZCI6InRvb2x1XzAxNjFYaFlQM0I1Zmc0VTFwc1QzcGNpUiIsInR5cGUiOiJ0b29sX2NhbGwifSx7Im5hbWUiOiJ0YXNrIiwiYXJncyI6eyJzdWJhZ2VudF90eXBlIjoicmVzZWFyY2hlciIsImRlc2NyaXB0aW9uIjoiUmVzZWFyY2ggYW5kIGNvbXBhcmUgQ2xpY2tIb3VzZSB2cyBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IChPVEVMKSBhZ2VudCBzcGFucyBhdCA1MCwwMDAgc3BhbnMgcGVyIHNlY29uZC5cblxuRm9jdXMgb246XG4xLiBXcml0ZSB0aHJvdWdocHV0IGNhcGFiaWxpdGllcy4uLiJ9LCJpZCI6InRvb2x1XzAxUEx5bzhUS0tUcFhSNGZwOTZEbjkzVyIsInR5cGUiOiJ0b29sX2NhbGwifV0sInVzYWdlX21ldGFkYXRhIjp7ImlucHV0X3Rva2VucyI6MzMzMiwib3V0cHV0X3Rva2VucyI6NDY3LCJ0b3RhbF90b2tlbnMiOjM3OTksImlucHV0X3Rva2VuX2RldGFpbHMiOnsiY2FjaGVfcmVhZCI6MCwiY2FjaGVfY3JlYXRpb24iOjMzMjl9LCJvdXRwdXRfdG9rZW5fZGV0YWlscyI6eyJyZWFzb25pbmciOjB9fSwiaW52YWxpZF90b29sX2NhbGxzIjpbXX19fV1dLCJsbG1fb3V0cHV0Ijp7InRva2VuX3VzYWdlIjp7ImNvbXBsZXRpb25fdG9rZW5zIjo0NjcsInByb21wdF90b2tlbnMiOjMzMzIsInRvdGFsX3Rva2VucyI6Mzc5OSwiY29tcGxldGlvbl90b2tlbnNfZGV0YWlscyI6eyJhY2NlcHRlZF9wcmVkaWN0aW9uX3Rva2VucyI6bnVsbCwiYXVkaW9fdG9rZW5zIjpudWxsLCJyZWFzb25pbmdfdG9rZW5zIjowLCJyZWplY3RlZF9wcmVkaWN0aW9uX3Rva2VucyI6bnVsbCwidGV4dF90b2tlbnMiOjQ2N30sInByb21wdF90b2tlbnNfZGV0YWlscyI6eyJhdWRpb190b2tlbnMiOm51bGwsImNhY2hlX3dyaXRlX3Rva2VucyI6MzMyOSwiY2FjaGVkX3Rva2VucyI6MCwiaW1hZ2VfdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6MywiY2FjaGVfY3JlYXRpb25fdG9rZW5zIjozMzI5LCJjYWNoZV9jcmVhdGlvbl90b2tlbl9kZXRhaWxzIjp7ImVwaGVtZXJhbF81bV9pbnB1dF90b2tlbnMiOjMzMjksImVwaGVtZXJhbF8xaF9pbnB1dF90b2tlbnMiOjB9fSwiY2FjaGVfY3JlYXRpb25faW5wdXRfdG9rZW5zIjozMzI5LCJjYWNoZV9yZWFkX2lucHV0X3Rva2VucyI6MCwiaW5mZXJlbmNlX2dlbyI6Im5vdF9hdmFpbGFibGUiLCJzZXJ2aWNlX3RpZXIiOiJzdGFuZGFyZCJ9LCJtb2RlbF9wcm92aWRlciI6Im9wZW5haSIsIm1vZGVsX25hbWUiOiJjbGF1ZGUtc29ubmV0LTQtNSIsInN5c3RlbV9maW5nZXJwcmludCI6bnVsbCwiaWQiOiJjaGF0Y21wbC00MDc3YmIzNi05MzgwLTRhM2ItOTQ4MS0yNDU3MDBjZWYwOWEifSwicnVuIjpudWxsLCJ0eXBlIjoiTExNUmVzdWx0In0=" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "zwThqgPzRPo=", + "parentSpanId": "g0UfMjWEf2w=", + "name": "FilesystemMiddleware.wrap_model_call", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742989379030016", + "endTimeUnixNano": "1790742998895730944", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chain" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "e30=" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "FilesystemMiddleware.wrap_model_call" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "deepagents" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "2" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "model" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"branch:to:model\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_pull\",\"model\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "model:9abb6d12-32f9-4289-15b6-36ac41ba926c" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJvdXRwdXQiOnsicmVzdWx0IjpbeyJjb250ZW50IjoiSSdsbCBoZWxwIHlvdSBkZWNpZGUgYmV0d2VlbiBDbGlja0hvdXNlIGFuZCBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IHNwYW5zIGF0IDUwayBzcGFucy9zZWMuIExldCBtZSByZXNlYXJjaCB0aGlzIHN5c3RlbWF0aWNhbGx5LiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7InJlZnVzYWwiOm51bGx9LCJyZXNwb25zZV9tZXRhZGF0YSI6eyJ0b2tlbl91c2FnZSI6eyJjb21wbGV0aW9uX3Rva2VucyI6NDY3LCJwcm9tcHRfdG9rZW5zIjozMzMyLCJ0b3RhbF90b2tlbnMiOjM3OTksImNvbXBsZXRpb25fdG9rZW5zX2RldGFpbHMiOnsiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsImF1ZGlvX3Rva2VucyI6bnVsbCwicmVhc29uaW5nX3Rva2VucyI6MCwicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjo0Njd9LCJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOnsiYXVkaW9fdG9rZW5zIjpudWxsLCJjYWNoZV93cml0ZV90b2tlbnMiOjMzMjksImNhY2hlZF90b2tlbnMiOjAsImltYWdlX3Rva2VucyI6bnVsbCwidGV4dF90b2tlbnMiOjMsImNhY2hlX2NyZWF0aW9uX3Rva2VucyI6MzMyOSwiY2FjaGVfY3JlYXRpb25fdG9rZW5fZGV0YWlscyI6eyJlcGhlbWVyYWxfNW1faW5wdXRfdG9rZW5zIjozMzI5LCJlcGhlbWVyYWxfMWhfaW5wdXRfdG9rZW5zIjowfX0sImNhY2hlX2NyZWF0aW9uX2lucHV0X3Rva2VucyI6MzMyOSwiY2FjaGVfcmVhZF9pbnB1dF90b2tlbnMiOjAsImluZmVyZW5jZV9nZW8iOiJub3RfYXZhaWxhYmxlIiwic2VydmljZV90aWVyIjoic3RhbmRhcmQifSwibW9kZWxfcHJvdmlkZXIiOiJvcGVuYWkiLCJtb2RlbF9uYW1lIjoiY2xhdWRlLXNvbm5ldC00LTUiLCJzeXN0ZW1fZmluZ2VycHJpbnQiOm51bGwsImlkIjoiY2hhdGNtcGwtNDA3N2JiMzYtOTM4MC00YTNiLTk0ODEtMjQ1NzAwY2VmMDlhIiwiZmluaXNoX3JlYXNvbiI6InRvb2xfY2FsbHMiLCJsb2dwcm9icyI6bnVsbH0sInR5cGUiOiJhaSIsIm5hbWUiOiJkZWVwX3Jlc2VhcmNoX2FnZW50IiwiaWQiOiJsY19ydW4tLTAxYTBmMDk5LThhNDctN2U0Mi1hNWY0LTVjNDdkZTNkMWNlYy0wIiwidG9vbF9jYWxscyI6W3sibmFtZSI6IndyaXRlX2ZpbGUiLCJhcmdzIjp7ImZpbGVfcGF0aCI6Ii90bXAvcmVzZWFyY2hfdG9kb3MubWQiLCJjb250ZW50IjoiIyBSZXNlYXJjaCBQbGFuOiBDbGlja0hvdXNlIHZzIFBvc3RncmVzIGZvciBPVEVMIFNwYW5zICg1MGsvc2VjKVxuXG4jIyBUYXNrc1xuLSBbIF0gUmVzZWFyY2ggQ2xpY2tIb3VzZSBhbmQgUG9zdGdyZXMgY2FwYWJpbGl0aWVzIGZvciBoaWdoLXZvbHVtZSB0aW1lLXNlcmllcyBkYXRhXG4uLi4ifSwiaWQiOiJ0b29sdV8wMTYxWGhZUDNCNWZnNFUxcHNUM3BjaVIiLCJ0eXBlIjoidG9vbF9jYWxsIn0seyJuYW1lIjoidGFzayIsImFyZ3MiOnsic3ViYWdlbnRfdHlwZSI6InJlc2VhcmNoZXIiLCJkZXNjcmlwdGlvbiI6IlJlc2VhcmNoIGFuZCBjb21wYXJlIENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSAoT1RFTCkgYWdlbnQgc3BhbnMgYXQgNTAsMDAwIHNwYW5zIHBlciBzZWNvbmQuXG5cbkZvY3VzIG9uOlxuMS4gV3JpdGUgdGhyb3VnaHB1dCBjYXBhYmlsaXRpZXMuLi4ifSwiaWQiOiJ0b29sdV8wMVBMeW84VEtLVHBYUjRmcDk2RG45M1ciLCJ0eXBlIjoidG9vbF9jYWxsIn1dLCJpbnZhbGlkX3Rvb2xfY2FsbHMiOltdLCJ1c2FnZV9tZXRhZGF0YSI6eyJpbnB1dF90b2tlbnMiOjMzMzIsIm91dHB1dF90b2tlbnMiOjQ2NywidG90YWxfdG9rZW5zIjozNzk5LCJpbnB1dF90b2tlbl9kZXRhaWxzIjp7ImNhY2hlX3JlYWQiOjAsImNhY2hlX2NyZWF0aW9uIjozMzI5fSwib3V0cHV0X3Rva2VuX2RldGFpbHMiOnsicmVhc29uaW5nIjowfX19XSwic3RydWN0dXJlZF9yZXNwb25zZSI6bnVsbH19" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "svs6j1ovzgE=", + "parentSpanId": "Vt73x+GSQ0o=", + "name": "task", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742998900896000", + "endTimeUnixNano": "1790743034076956160", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "tool" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "task" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "task" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "toolu_01PLyo8TKKTpXR4fp96Dn93W" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "deepagents" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "3" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "tools" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"__pregel_push\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_push\",1,false]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "langsmith.span.tags", + "value": { + "stringValue": "seq:step:1" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJzdWJhZ2VudF90eXBlIjoicmVzZWFyY2hlciIsImRlc2NyaXB0aW9uIjoiUmVzZWFyY2ggYW5kIGNvbXBhcmUgQ2xpY2tIb3VzZSB2cyBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IChPVEVMKSBhZ2VudCBzcGFucyBhdCA1MCwwMDAgc3BhbnMgcGVyIHNlY29uZC5cblxuRm9jdXMgb246XG4xLiBXcml0ZSB0aHJvdWdocHV0IGNhcGFiaWxpdGllcy4uLiJ9" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJvdXRwdXQiOnsiZ3JhcGgiOm51bGwsInVwZGF0ZSI6eyJmaWxlcyI6e30sIm1lc3NhZ2VzIjpbeyJjb250ZW50IjoiQmFzZWQgb24gbXkgcmVzZWFyY2gsIGhlcmUncyBhIGNvbXByZWhlbnNpdmUgY29tcGFyaXNvbiBvZiAqKkNsaWNrSG91c2UgdnMgUG9zdGdyZXMqKiBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IHNwYW5zIGF0IDUwLDAwMCBzcGFucy9zZWNvbmQ6XG5cbiMjICoqMS4gV3JpdGUgVGhyLi4uIiwiYWRkaXRpb25hbF9rd2FyZ3MiOnt9LCJyZXNwb25zZV9tZXRhZGF0YSI6e30sInR5cGUiOiJ0b29sIiwidG9vbF9jYWxsX2lkIjoidG9vbHVfMDFQTHlvOFRLS1RwWFI0ZnA5NkRuOTNXIiwic3RhdHVzIjoic3VjY2VzcyJ9XX0sInJlc3VtZSI6bnVsbCwiZ290byI6W119fQ==" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "gUmbSS/ZP4U=", + "parentSpanId": "svs6j1ovzgE=", + "name": "researcher", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742998901422080", + "endTimeUnixNano": "1790743034076699904", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "researcher" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "langchain_create_agent" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "researcher" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "3" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "tools" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"__pregel_push\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_push\",1,false]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.ls_agent_type", + "value": { + "stringValue": "subagent" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJmaWxlcyI6e30sIm1lc3NhZ2VzIjpbeyJjb250ZW50IjoiUmVzZWFyY2ggYW5kIGNvbXBhcmUgQ2xpY2tIb3VzZSB2cyBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IChPVEVMKSBhZ2VudCBzcGFucyBhdCA1MCwwMDAgc3BhbnMgcGVyIHNlY29uZC5cblxuRm9jdXMgb246XG4xLiBXcml0ZSB0aHJvdWdocHV0IGNhcGFiaWxpdGllcy4uLiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7fSwicmVzcG9uc2VfbWV0YWRhdGEiOnt9LCJ0eXBlIjoiaHVtYW4iLCJpZCI6ImFmOGRiNzQ5LTBiNTYtNGEzMi1hZGZlLTdmYzViOTRmZDAwMyJ9XX0=" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJtZXNzYWdlcyI6W3siY29udGVudCI6IlJlc2VhcmNoIGFuZCBjb21wYXJlIENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSAoT1RFTCkgYWdlbnQgc3BhbnMgYXQgNTAsMDAwIHNwYW5zIHBlciBzZWNvbmQuXG5cbkZvY3VzIG9uOlxuMS4gV3JpdGUgdGhyb3VnaHB1dCBjYXBhYmlsaXRpZXMuLi4iLCJhZGRpdGlvbmFsX2t3YXJncyI6e30sInJlc3BvbnNlX21ldGFkYXRhIjp7fSwidHlwZSI6Imh1bWFuIiwiaWQiOiJhZjhkYjc0OS0wYjU2LTRhMzItYWRmZS03ZmM1Yjk0ZmQwMDMifSx7ImNvbnRlbnQiOiJJJ2xsIHJlc2VhcmNoIHRoZSBjb21wYXJpc29uIGJldHdlZW4gQ2xpY2tIb3VzZSBhbmQgUG9zdGdyZXMgZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSBzcGFucyBhdCBoaWdoIHZvbHVtZS4iLCJhZGRpdGlvbmFsX2t3YXJncyI6eyJyZWZ1c2FsIjpudWxsfSwicmVzcG9uc2VfbWV0YWRhdGEiOnsidG9rZW5fdXNhZ2UiOnsiY29tcGxldGlvbl90b2tlbnMiOjQyNywicHJvbXB0X3Rva2VucyI6Mjk4NiwidG90YWxfdG9rZW5zIjozNDEzLCJjb21wbGV0aW9uX3Rva2Vuc19kZXRhaWxzIjp7ImFjY2VwdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjpudWxsLCJhdWRpb190b2tlbnMiOm51bGwsInJlYXNvbmluZ190b2tlbnMiOjAsInJlamVjdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6NDI3fSwicHJvbXB0X3Rva2Vuc19kZXRhaWxzIjp7ImF1ZGlvX3Rva2VucyI6bnVsbCwiY2FjaGVfd3JpdGVfdG9rZW5zIjoyOTgzLCJjYWNoZWRfdG9rZW5zIjowLCJpbWFnZV90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjozLCJjYWNoZV9jcmVhdGlvbl90b2tlbnMiOjI5ODMsImNhY2hlX2NyZWF0aW9uX3Rva2VuX2RldGFpbHMiOnsiZXBoZW1lcmFsXzVtX2lucHV0X3Rva2VucyI6Mjk4MywiZXBoZW1lcmFsXzFoX2lucHV0X3Rva2VucyI6MH19LCJjYWNoZV9jcmVhdGlvbl9pbnB1dF90b2tlbnMiOjI5ODMsImNhY2hlX3JlYWRfaW5wdXRfdG9rZW5zIjowLCJpbmZlcmVuY2VfZ2VvIjoibm90X2F2YWlsYWJsZSIsInNlcnZpY2VfdGllciI6InN0YW5kYXJkIn0sIm1vZGVsX3Byb3ZpZGVyIjoib3BlbmFpIiwibW9kZWxfbmFtZSI6ImNsYXVkZS1zb25uZXQtNC01Iiwic3lzdGVtX2ZpbmdlcnByaW50IjpudWxsLCJpZCI6ImNoYXRjbXBsLWFhYWE0Yjc4LTE3ZGMtNDM2NC04ZmE1LTJkODMzNjlmMWRiYyIsImZpbmlzaF9yZWFzb24iOiJ0b29sX2NhbGxzIiwibG9ncHJvYnMiOm51bGx9LCJ0eXBlIjoiYWkiLCJuYW1lIjoicmVzZWFyY2hlciIsImlkIjoibGNfcnVuLS0wMWEwZjA5OS1hZjdlLTc5ZTAtYTMzMy03MDdjMzQ5N2M3MzAtMCIsInRvb2xfY2FsbHMiOlt7Im5hbWUiOiJzZWFyY2hfZG9jcyIsImFyZ3MiOnsicXVlcnkiOiJDbGlja0hvdXNlIFBvc3RncmVzIE9wZW5UZWxlbWV0cnkgT1RFTCBzcGFucyBwZXJmb3JtYW5jZSBjb21wYXJpc29uIn0sImlkIjoidG9vbHVfMDFKc2pIRkZmcHN3NG9wbUs5VVppOFZOIiwidHlwZSI6InRvb2xfY2FsbCJ9LHsibmFtZSI6InNlYXJjaF9kb2NzIiwiYXJncyI6eyJxdWVyeSI6IkNsaWNrSG91c2Ugd3JpdGUgdGhyb3VnaHB1dCA1MDAwMCBzcGFucyBwZXIgc2Vjb25kIHRlbGVtZXRyeSJ9LCJpZCI6InRvb2x1XzAxS05ZcUhKS2kzcExlZU1RaEc1VDl1ZSIsInR5cGUiOiJ0b29sX2NhbGwifSx7Im5hbWUiOiJzZWFyY2hfZG9jcyIsImFyZ3MiOnsicXVlcnkiOiJQb3N0Z3JlcyB2cyBDbGlja0hvdXNlIG9ic2VydmFiaWxpdHkgbWV0cmljcyB0cmFjZXMifSwiaWQiOiJ0b29sdV8wMUpoRjh6NDQ0U1dVM0VXM2hQUUtkMlciLCJ0eXBlIjoidG9vbF9jYWxsIn0seyJuYW1lIjoic2VhcmNoX2RvY3MiLCJhcmdzIjp7InF1ZXJ5IjoiQ2xpY2tIb3VzZSBpbnNlcnQgcGVyZm9ybWFuY2UgYmF0Y2ggd3JpdGVzIHN1c3RhaW5lZCB0aHJvdWdocHV0In0sImlkIjoidG9vbHVfMDFXdXFyNTZKVHhDSllRUFMxWkZQbkg2IiwidHlwZSI6InRvb2xfY2FsbCJ9XSwiaW52YWxpZF90b29sX2NhbGxzIjpbXSwidXNhZ2VfbWV0YWRhdGEiOnsiaW5wdXRfdG9rZW5zIjoyOTg2LCJvdXRwdXRfdG9rZW5zIjo0MjcsInRvdGFsX3Rva2VucyI6MzQxMywiaW5wdXRfdG9rZW5fZGV0YWlscyI6eyJjYWNoZV9yZWFkIjowLCJjYWNoZV9jcmVhdGlvbiI6Mjk4M30sIm91dHB1dF90b2tlbl9kZXRhaWxzIjp7InJlYXNvbmluZyI6MH19fSx7ImNvbnRlbnQiOiJObyByZXN1bHRzLiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7fSwicmVzcG9uc2VfbWV0YWRhdGEiOnt9LCJ0eXBlIjoidG9vbCIsIm5hbWUiOiJzZWFyY2hfZG9jcyIsImlkIjoiYTNhMDQxMmUtMWMxNS00ODk3LThjMmQtZGM0NmMwYmRlYzM2IiwidG9vbF9jYWxsX2lkIjoidG9vbHVfMDFVQmFYd0JQTmRxUkhHYmJhbmdLTFpVIiwic3RhdHVzIjoic3VjY2VzcyJ9LHsiY29udGVudCI6IkJhc2VkIG9uIG15IHJlc2VhcmNoLCBoZXJlJ3MgYSBjb21wcmVoZW5zaXZlIGNvbXBhcmlzb24gb2YgKipDbGlja0hvdXNlIHZzIFBvc3RncmVzKiogZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSBzcGFucyBhdCA1MCwwMDAgc3BhbnMvc2Vjb25kOlxuXG4jIyAqKjEuIFdyaXRlIFRoci4uLiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7InJlZnVzYWwiOm51bGx9LCJyZXNwb25zZV9tZXRhZGF0YSI6eyJ0b2tlbl91c2FnZSI6eyJjb21wbGV0aW9uX3Rva2VucyI6NzAwLCJwcm9tcHRfdG9rZW5zIjo1NTM2LCJ0b3RhbF90b2tlbnMiOjYyMzYsImNvbXBsZXRpb25fdG9rZW5zX2RldGFpbHMiOnsiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsImF1ZGlvX3Rva2VucyI6bnVsbCwicmVhc29uaW5nX3Rva2VucyI6MCwicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjo3MDB9LCJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOnsiYXVkaW9fdG9rZW5zIjpudWxsLCJjYWNoZV93cml0ZV90b2tlbnMiOjQzMiwiY2FjaGVkX3Rva2VucyI6NTA5NywiaW1hZ2VfdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6NywiY2FjaGVfY3JlYXRpb25fdG9rZW5zIjo0MzIsImNhY2hlX2NyZWF0aW9uX3Rva2VuX2RldGFpbHMiOnsiZXBoZW1lcmFsXzVtX2lucHV0X3Rva2VucyI6NDMyLCJlcGhlbWVyYWxfMWhfaW5wdXRfdG9rZW5zIjowfX0sImNhY2hlX2NyZWF0aW9uX2lucHV0X3Rva2VucyI6NDMyLCJjYWNoZV9yZWFkX2lucHV0X3Rva2VucyI6NTA5NywiaW5mZXJlbmNlX2dlbyI6Im5vdF9hdmFpbGFibGUiLCJzZXJ2aWNlX3RpZXIiOiJzdGFuZGFyZCJ9LCJtb2RlbF9wcm92aWRlciI6Im9wZW5haSIsIm1vZGVsX25hbWUiOiJjbGF1ZGUtc29ubmV0LTQtNSIsInN5c3RlbV9maW5nZXJwcmludCI6bnVsbCwiaWQiOiJjaGF0Y21wbC0zYzIwZTgwOC05YjE2LTQ0MjctOTk0Zi01Y2U3ZThiMWI5NGQiLCJmaW5pc2hfcmVhc29uIjoibGVuZ3RoIiwibG9ncHJvYnMiOm51bGx9LCJ0eXBlIjoiYWkiLCJuYW1lIjoicmVzZWFyY2hlciIsImlkIjoibGNfcnVuLS0wMWEwZjA5OS1mYmMyLTc5NjMtOWViYy1kYWUzNzZkYmJhMzktMCIsInRvb2xfY2FsbHMiOltdLCJpbnZhbGlkX3Rvb2xfY2FsbHMiOltdLCJ1c2FnZV9tZXRhZGF0YSI6eyJpbnB1dF90b2tlbnMiOjU1MzYsIm91dHB1dF90b2tlbnMiOjcwMCwidG90YWxfdG9rZW5zIjo2MjM2LCJpbnB1dF90b2tlbl9kZXRhaWxzIjp7ImNhY2hlX3JlYWQiOjUwOTcsImNhY2hlX2NyZWF0aW9uIjo0MzJ9LCJvdXRwdXRfdG9rZW5fZGV0YWlscyI6eyJyZWFzb25pbmciOjB9fX1dLCJmaWxlcyI6e319" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "/mLyrQOgEWw=", + "parentSpanId": "SUm+6tN4+TU=", + "name": "search_docs", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790743004976721920", + "endTimeUnixNano": "1790743004977214208", + "attributes": [ + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "tool" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "search_docs" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search_docs" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "toolu_01JsjHFFfpsw4opmK9UZi8VN" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "langchain_create_agent" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "researcher" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "3" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "tools" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"__pregel_push\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_push\",0,false]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc|tools:49218779-253b-df87-734a-cfd23327bc5d" + } + }, + { + "key": "langsmith.metadata.checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.ls_agent_type", + "value": { + "stringValue": "subagent" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "langsmith.span.tags", + "value": { + "stringValue": "seq:step:1" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJxdWVyeSI6IkNsaWNrSG91c2UgUG9zdGdyZXMgT3BlblRlbGVtZXRyeSBPVEVMIHNwYW5zIHBlcmZvcm1hbmNlIGNvbXBhcmlzb24ifQ==" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJvdXRwdXQiOnsiY29udGVudCI6IkNsaWNrSG91c2UgaW5nZXN0cyAxTSsgcm93cy9zIHBlciBub2RlIHdpdGggYmF0Y2hlZCBpbnNlcnRzOyB1c2UgTWVyZ2VUcmVlIG9yZGVyZWQgYnkgKHRlbmFudCwgc2VydmljZSwgdGltZSkgYW5kIGEgYmxvb20gZmlsdGVyIGluZGV4IG9uIFRyYWNlSWQuXG5Qb3N0Z3JlcyBoYW5kLi4uIiwiYWRkaXRpb25hbF9rd2FyZ3MiOnt9LCJyZXNwb25zZV9tZXRhZGF0YSI6e30sInR5cGUiOiJ0b29sIiwibmFtZSI6InNlYXJjaF9kb2NzIiwidG9vbF9jYWxsX2lkIjoidG9vbHVfMDFKc2pIRkZmcHN3NG9wbUs5VVppOFZOIiwic3RhdHVzIjoic3VjY2VzcyJ9fQ==" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_litellm/tracing/test_decode.py b/tests/test_litellm/tracing/test_decode.py new file mode 100644 index 00000000000..168ff2bf7fb --- /dev/null +++ b/tests/test_litellm/tracing/test_decode.py @@ -0,0 +1,313 @@ +""" +Tests for OTLP decode + normalization (litellm/tracing/decode.py). + +The fixture is a trimmed real export from a Deep Agents run (LangSmith OTEL mode): +deep_research_agent -> task (tool) -> researcher (subagent) -> search_docs (tool). +""" + +import gzip +import json +from pathlib import Path +from unittest.mock import patch + +import pytest +from google.protobuf.json_format import Parse +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue +from opentelemetry.proto.trace.v1.trace_pb2 import ResourceSpans, ScopeSpans, Span, Status + +from litellm.tracing import decode +from litellm.tracing.decode import decode_otlp, encode_otlp_response + +pytestmark = pytest.mark.requires_rust_extension + +FIXTURE = Path(__file__).parent / "fixtures" / "langsmith_deep_agent_export.json" +TRACE_ID = "4bad42b84e9de3ba46fc870185f8f023" + + +def _fixture_json() -> bytes: + return FIXTURE.read_bytes() + + +def _fixture_protobuf() -> bytes: + request = ExportTraceServiceRequest() + Parse(_fixture_json().decode(), request) + return request.SerializeToString() + + +@pytest.fixture +def rows_by_name() -> dict: + rows = decode_otlp(_fixture_json(), "application/json") + return {r["SpanName"]: r for r in rows} + + +def _kv(key: str, value: str | int) -> KeyValue: + if isinstance(value, int): + return KeyValue(key=key, value=AnyValue(int_value=value)) + return KeyValue(key=key, value=AnyValue(string_value=value)) + + +def _export(*spans: Span, service: str = "svc", scope: str = "test") -> bytes: + resource_spans = ResourceSpans(scope_spans=[ScopeSpans(spans=list(spans))]) + resource_spans.resource.attributes.append(_kv("service.name", service)) + resource_spans.scope_spans[0].scope.name = scope + return ExportTraceServiceRequest(resource_spans=[resource_spans]).SerializeToString() + + +def _span(name: str, span_id: bytes, parent: bytes = b"", **attributes: str | int) -> Span: + return Span( + trace_id=bytes.fromhex(TRACE_ID), + span_id=span_id, + parent_span_id=parent, + name=name, + start_time_unix_nano=1_000, + end_time_unix_nano=5_000, + attributes=[_kv(k.replace("__", "."), v) for k, v in attributes.items()], + ) + + +# ---------------------------------------------------------------- LangSmith / Deep Agents fixture + + +def test_classifies_every_langsmith_span(rows_by_name): + assert {name: r["ObservationType"] for name, r in rows_by_name.items()} == { + "deep_research_agent": "agent", + "ChatOpenAI": "llm", + "FilesystemMiddleware.wrap_model_call": "framework", + "task": "tool", + "researcher": "agent", + "search_docs": "tool", + } + + +def test_agent_name_is_the_enclosing_agent(rows_by_name): + assert rows_by_name["task"]["AgentName"] == "deep_research_agent" + assert rows_by_name["ChatOpenAI"]["AgentName"] == "deep_research_agent" + assert rows_by_name["researcher"]["AgentName"] == "researcher" + assert rows_by_name["search_docs"]["AgentName"] == "researcher" + + +def test_subagent_is_nested_under_task_tool(rows_by_name): + assert rows_by_name["researcher"]["ParentSpanId"] == rows_by_name["task"]["SpanId"] + assert rows_by_name["deep_research_agent"]["ParentSpanId"] == "" + + +def test_llm_span_carries_litellm_request_id_model_and_tokens(rows_by_name): + llm = rows_by_name["ChatOpenAI"] + assert llm["LiteLLMRequestId"] == "chatcmpl-4077bb36-9380-4a3b-9481-245700cef09a" + assert llm["Model"] == "claude-sonnet-4-5" + assert (llm["InputTokens"], llm["OutputTokens"]) == (3332, 467) + + +def test_llm_input_output_are_normalized_messages(rows_by_name): + llm = rows_by_name["ChatOpenAI"] + messages = json.loads(llm["Input"]) + assert [m["role"] for m in messages][:2] == ["system", "user"] + assert "research lead" in messages[0]["content"] + output = json.loads(llm["Output"]) + assert output["role"] == "assistant" + assert output["tool_calls"][0]["name"] + + +@pytest.mark.parametrize("completion", ["{}", '{"generations": []}', '{"generations": [[{}]]}']) +def test_incomplete_langsmith_completion_preserves_the_export(completion): + span = _span( + "ChatOpenAI", + b"\x03" * 8, + b"\x02" * 8, + langsmith__span__kind="llm", + gen_ai__prompt='{"messages": [[{"kwargs": {"type": "human", "content": "hi"}}]]}', + gen_ai__completion=completion, + ) + rows = decode_otlp(_export(span, scope="langsmith"), "application/x-protobuf") + assert len(rows) == 1 + assert json.loads(rows[0]["Input"])[0]["content"] == "hi" + assert rows[0]["Output"] == completion + + +def test_task_tool_output_is_subagent_final_message_text(rows_by_name): + task = rows_by_name["task"] + assert json.loads(task["Input"])["subagent_type"] == "researcher" + assert task["Output"].startswith("Based on my research") + assert not task["Output"].startswith("{") + + +def test_agent_input_output(rows_by_name): + root = rows_by_name["deep_research_agent"] + assert json.loads(root["Input"]) == [ + {"role": "user", "content": "Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?"} + ] + assert json.loads(root["Output"])["role"] == "assistant" + + +def test_plain_tool_input_output(rows_by_name): + tool = rows_by_name["search_docs"] + assert json.loads(tool["Input"]) == {"query": "ClickHouse Postgres OpenTelemetry OTEL spans performance comparison"} + assert tool["Output"].startswith("ClickHouse ingests") + + +def test_heavy_attributes_are_lifted_out_of_span_attributes(rows_by_name): + for row in rows_by_name.values(): + assert not set(row["SpanAttributes"]) & decode._HEAVY_ATTRIBUTES + assert rows_by_name["ChatOpenAI"]["SpanAttributes"]["langsmith.span.kind"] == "llm" + + +def test_ids_are_hex_and_resource_is_kept(rows_by_name): + root = rows_by_name["deep_research_agent"] + assert root["TraceId"] == TRACE_ID + assert root["SpanId"] == "5e79f3b5b504985e" + assert root["ServiceName"] == "agent-demo" + assert root["ScopeName"] == "langsmith" + assert root["SpanKind"] == "SPAN_KIND_INTERNAL" + assert root["StatusCode"] == "STATUS_CODE_OK" + assert root["Duration"] > 0 + + +def test_protobuf_and_json_decode_identically(): + from_json = decode_otlp(_fixture_json(), "application/json") + from_protobuf = decode_otlp(_fixture_protobuf(), "application/x-protobuf") + assert from_json == from_protobuf + assert len(from_json) == 6 + + +def test_content_type_defaults_to_protobuf(): + assert len(decode_otlp(_fixture_protobuf(), None)) == 6 + + +@pytest.mark.parametrize("content_encoding", ["gzip", None]) +def test_gzip_body_by_header_or_magic_bytes(content_encoding): + rows = decode_otlp(gzip.compress(_fixture_protobuf()), "application/x-protobuf", content_encoding) + assert len(rows) == 6 + + +def test_long_values_are_truncated_with_marker(): + with patch.object(decode, "OTLP_MAX_ATTRIBUTE_VALUE_BYTES", 100): + rows = {r["SpanName"]: r for r in decode_otlp(_fixture_json(), "application/json")} + task = rows["task"] + assert "…[truncated " in task["Input"] + assert task["Input"].encode().startswith(task["Input"].split("…")[0].encode()) + assert len(task["Input"].split("…")[0].encode()) <= 100 + + +# ---------------------------------------------------------------- status / exceptions + + +def test_exception_event_fills_status_message(): + span = _span("get_customer_plan", b"\x01" * 8, b"\x02" * 8) + span.status.CopyFrom(Status(code=Status.STATUS_CODE_ERROR)) + event = span.events.add() + event.name = "exception" + event.attributes.extend( + [_kv("exception.type", "KeyError"), _kv("exception.message", "customer acme-404 not found")] + ) + (row,) = decode_otlp(_export(span)) + assert row["StatusCode"] == "STATUS_CODE_ERROR" + assert row["StatusMessage"] == "customer acme-404 not found" + + +def test_status_message_wins_over_exception_event(): + span = _span("tool", b"\x01" * 8, b"\x02" * 8) + span.status.CopyFrom(Status(code=Status.STATUS_CODE_ERROR, message="boom")) + event = span.events.add() + event.name = "exception" + event.attributes.append(_kv("exception.message", "other")) + (row,) = decode_otlp(_export(span)) + assert row["StatusMessage"] == "boom" + + +# ---------------------------------------------------------------- GenAI semconv / OpenInference + + +def test_genai_semconv_spans(): + root = _span( + "invoke_agent planner", b"\x01" * 8, gen_ai__operation__name="invoke_agent", gen_ai__agent__name="planner" + ) + chat = _span( + "chat gpt-4o", + b"\x02" * 8, + b"\x01" * 8, + gen_ai__operation__name="chat", + gen_ai__agent__name="planner", + gen_ai__request__model="gpt-4o", + gen_ai__response__id="chatcmpl-abc", + gen_ai__usage__input_tokens=12, + gen_ai__usage__output_tokens=3, + gen_ai__input__messages='[{"role":"user","content":"hi"}]', + gen_ai__output__messages='[{"role":"assistant","content":"hello"}]', + ) + tool = _span( + "execute_tool search", + b"\x03" * 8, + b"\x01" * 8, + gen_ai__operation__name="execute_tool", + gen_ai__tool__call__arguments='{"q":"x"}', + gen_ai__tool__call__result="found", + ) + rows = {r["SpanName"]: r for r in decode_otlp(_export(root, chat, tool))} + assert rows["invoke_agent planner"]["ObservationType"] == "agent" + assert rows["invoke_agent planner"]["AgentName"] == "planner" + llm = rows["chat gpt-4o"] + assert (llm["ObservationType"], llm["Model"], llm["LiteLLMRequestId"]) == ("llm", "gpt-4o", "chatcmpl-abc") + assert (llm["InputTokens"], llm["OutputTokens"]) == (12, 3) + assert json.loads(llm["Input"])[0]["content"] == "hi" + assert "gen_ai.input.messages" not in llm["SpanAttributes"] + assert (rows["execute_tool search"]["ObservationType"], rows["execute_tool search"]["Output"]) == ("tool", "found") + + +def test_openinference_spans(): + root = _span("agent", b"\x01" * 8, openinference__span__kind="AGENT", agent__name="writer", input__value="task") + llm = _span( + "llm", + b"\x02" * 8, + b"\x01" * 8, + openinference__span__kind="LLM", + llm__model_name="claude-sonnet-4-5", + llm__token_count__prompt=40, + llm__token_count__completion=8, + input__value="prompt", + output__value="answer", + ) + chain = _span("retriever", b"\x03" * 8, b"\x01" * 8, openinference__span__kind="RETRIEVER") + rows = {r["SpanName"]: r for r in decode_otlp(_export(root, llm, chain))} + assert (rows["agent"]["ObservationType"], rows["agent"]["AgentName"], rows["agent"]["Input"]) == ( + "agent", + "writer", + "task", + ) + assert rows["llm"]["ObservationType"] == "llm" + assert (rows["llm"]["Model"], rows["llm"]["InputTokens"], rows["llm"]["OutputTokens"]) == ( + "claude-sonnet-4-5", + 40, + 8, + ) + assert (rows["llm"]["Input"], rows["llm"]["Output"]) == ("prompt", "answer") + assert "input.value" not in rows["llm"]["SpanAttributes"] + assert rows["retriever"]["ObservationType"] == "chain" + + +def test_non_string_attribute_values_are_stringified(): + span = _span("root", b"\x01" * 8) + span.attributes.extend( + [ + KeyValue(key="flag", value=AnyValue(bool_value=True)), + KeyValue(key="ratio", value=AnyValue(double_value=0.5)), + KeyValue(key="raw", value=AnyValue(bytes_value=b"abc")), + ] + ) + array = KeyValue(key="list") + array.value.array_value.values.extend([AnyValue(string_value="a"), AnyValue(int_value=1)]) + span.attributes.append(array) + (row,) = decode_otlp(_export(span)) + assert row["SpanAttributes"]["flag"] == "true" + assert row["SpanAttributes"]["ratio"] == "0.5" + assert row["SpanAttributes"]["raw"] == "abc" + assert json.loads(row["SpanAttributes"]["list"]) == ["a", "1"] + + +# ---------------------------------------------------------------- helpers + + +def test_encode_otlp_response_matches_request_encoding(): + assert encode_otlp_response("application/json") == (b"{}", "application/json") + assert encode_otlp_response("application/x-protobuf") == (b"", "application/x-protobuf") + assert encode_otlp_response(None) == (b"", "application/x-protobuf") diff --git a/tests/test_litellm/tracing/test_receiver.py b/tests/test_litellm/tracing/test_receiver.py new file mode 100644 index 00000000000..d492844db79 --- /dev/null +++ b/tests/test_litellm/tracing/test_receiver.py @@ -0,0 +1,117 @@ +""" +Tests for TraceReceiver.ingest (litellm/tracing/receiver.py) with a fake store. +""" + +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue +from opentelemetry.proto.trace.v1.trace_pb2 import ResourceSpans, ScopeSpans, Span + +from litellm.tracing import Tenant, TraceReceiver, TracingPayloadTooLargeError +from litellm.tracing import receiver as receiver_module +from litellm.tracing.types import TraceScope + +pytestmark = pytest.mark.requires_rust_extension + +FIXTURE = Path(__file__).parent / "fixtures" / "langsmith_deep_agent_export.json" +TENANT = Tenant(team_id="team-research", api_key_hash="hashed-key", org_id="org-1") + + +def _fake_store() -> MagicMock: + store = MagicMock() + store.insert_spans = AsyncMock() + store.get_trace = AsyncMock(return_value=None) + return store + + +def _spoofed_export() -> bytes: + """A client that tries to claim another team via resource attributes.""" + resource_spans = ResourceSpans(scope_spans=[ScopeSpans(spans=[Span(trace_id=b"\x01" * 16, span_id=b"\x02" * 8)])]) + resource_spans.resource.attributes.extend( + [ + KeyValue(key="service.name", value=AnyValue(string_value="svc")), + KeyValue(key="litellm.team_id", value=AnyValue(string_value="someone-elses-team")), + KeyValue(key="litellm.api_key_hash", value=AnyValue(string_value="someone-elses-key")), + ] + ) + return ExportTraceServiceRequest(resource_spans=[resource_spans]).SerializeToString() + + +@pytest.mark.asyncio +async def test_ingest_returns_span_count_and_writes_stamped_rows(): + store = _fake_store() + count = await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + assert count == 6 + (rows,) = store.insert_spans.await_args.args + assert len(rows) == 6 + for row in rows: + assert (row["TeamId"], row["ApiKeyHash"]) == ("team-research", "hashed-key") + assert row["ResourceAttributes"]["litellm.org_id"] == "org-1" + assert row["ResourceAttributes"]["service.name"] == "agent-demo" + + +@pytest.mark.asyncio +async def test_ingest_overwrites_client_supplied_tenant_attributes(): + store = _fake_store() + await TraceReceiver(store).ingest(_spoofed_export(), "application/x-protobuf", None, TENANT) + ((row,),) = store.insert_spans.await_args.args + assert row["TeamId"] == "team-research" + assert row["ResourceAttributes"]["litellm.team_id"] == "team-research" + assert row["ResourceAttributes"]["litellm.api_key_hash"] == "hashed-key" + + +@pytest.mark.asyncio +async def test_ingest_does_not_acknowledge_failed_clickhouse_write(): + store = _fake_store() + store.insert_spans.side_effect = RuntimeError("ClickHouse unavailable") + with pytest.raises(RuntimeError, match="ClickHouse unavailable"): + await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + store.insert_spans.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_ingest_rejects_oversized_encoded_batch(): + store = _fake_store() + store.insert_spans.side_effect = OverflowError("ClickHouse insert exceeds the encoded size limit") + with pytest.raises(TracingPayloadTooLargeError, match="encoded size limit"): + await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + + +@pytest.mark.asyncio +async def test_ingest_rejects_oversized_body(): + store = _fake_store() + with patch.object(receiver_module, "OTLP_MAX_BODY_BYTES", 10): + with pytest.raises(TracingPayloadTooLargeError): + await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + store.insert_spans.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_large_body_is_decoded_off_the_event_loop(): + store = _fake_store() + with ( + patch.object(receiver_module, "OTLP_OFFLOAD_DECODE_BYTES", 0), + patch.object(receiver_module.asyncio, "to_thread", wraps=receiver_module.asyncio.to_thread) as to_thread, + ): + count = await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + assert count == 6 + to_thread.assert_called_once() + + +@pytest.mark.asyncio +async def test_empty_export_writes_nothing(): + store = _fake_store() + assert await TraceReceiver(store).ingest(b"", "application/x-protobuf", None, TENANT) == 0 + store.insert_spans.assert_awaited_once_with(()) + + +@pytest.mark.asyncio +async def test_reads_delegate_to_store(): + store = _fake_store() + tracing = TraceReceiver(store) + scope: TraceScope = {"team_ids": ("team-research",), "api_key_hash": ""} + assert await tracing.get_trace("t1", scope) is None + store.get_trace.assert_awaited_once_with("t1", scope, "") diff --git a/tests/test_litellm/tracing/test_store.py b/tests/test_litellm/tracing/test_store.py new file mode 100644 index 00000000000..39ce3162073 --- /dev/null +++ b/tests/test_litellm/tracing/test_store.py @@ -0,0 +1,311 @@ +""" +Tests for the pure read-side helpers in litellm/tracing/store.py (no ClickHouse needed). +""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.tracing.store import ( + ClickHouseTraceStore, + agent_nodes, + decode_cursor, + encode_cursor, + span_from_row, + trace_from_rows, + trace_summary_from_row, +) +from litellm.tracing.types import TraceScope + +T0 = 1_790_742_989_000_000_000 # ns +MS = 1_000_000 + + +def _row( + span_id: str, + parent: str, + name: str, + type_: str, + agent: str, + start_ms: float = 0, + duration_ms: float = 10, + status: str = "STATUS_CODE_OK", + **extra: Any, +) -> dict[str, Any]: + return { + "span_id": span_id, + "parent_span_id": parent, + "name": name, + "type": type_, + "agent": agent, + "status": status, + "start_ns": T0 + int(start_ms * MS), + "duration_ns": int(duration_ms * MS), + "service": "agent-demo", + "input_preview": f"input of {name}", + "model": "", + "input_tokens": 0, + "output_tokens": 0, + "litellm_request_id": "", + **extra, + } + + +def _llm_row(span_id: str, parent: str, agent: str, request_id: str, start_ms: float = 1) -> dict: + return _row( + span_id, + parent, + "ChatOpenAI", + "llm", + agent, + start_ms=start_ms, + duration_ms=100, + model="claude-sonnet-4-5", + input_tokens=100, + output_tokens=20, + litellm_request_id=request_id, + ) + + +def _deep_agent_rows(researcher_invocations: int = 1) -> list[dict[str, Any]]: + """root agent -> llm, task tool -> researcher subagent (N times) -> llm + search_docs tool.""" + rows = [ + _row("root", "", "deep_research_agent", "agent", "deep_research_agent", duration_ms=1000), + _llm_row("llm-root", "root", "deep_research_agent", "chatcmpl-root"), + _row("task", "root", "task", "tool", "deep_research_agent", start_ms=200, duration_ms=700), + ] + for i in range(researcher_invocations): + rows += [ + _row(f"res-{i}", "task", "researcher", "agent", "researcher", start_ms=201, duration_ms=5), + _llm_row(f"res-llm-{i}", f"res-{i}", "researcher", f"chatcmpl-res-{i}", start_ms=202), + _row(f"res-tool-{i}", f"res-{i}", "search_docs", "tool", "researcher", start_ms=203, duration_ms=1), + _row(f"res-mw-{i}", f"res-{i}", "FilesystemMiddleware.wrap_model_call", "framework", "researcher"), + ] + return rows + + +# ---------------------------------------------------------------- trace_from_rows + + +def test_empty_rows_is_none(): + assert trace_from_rows("abc", []) is None + + +def test_llm_response_id_is_preserved_without_spend_enrichment(): + trace = trace_from_rows("t1", _deep_agent_rows()) + assert trace is not None + spans = {span["span_id"]: span for span in trace["spans"]} + assert spans["llm-root"]["litellm_request_id"] == "chatcmpl-root" + assert spans["task"]["litellm_request_id"] is None + assert "spend" not in trace["summary"] + + +def test_summary_totals(): + trace = trace_from_rows("t1", _deep_agent_rows()) + assert trace is not None + summary = trace["summary"] + assert summary["trace_id"] == "t1" + assert summary["name"] == "deep_research_agent" + assert summary["service"] == "agent-demo" + assert summary["input_preview"] == "input of deep_research_agent" + assert summary["status"] == "ok" + assert summary["span_count"] == 7 + assert summary["agent_count"] == 2 + assert summary["llm_calls"] == 2 + assert summary["tool_calls"] == 2 + assert summary["error_count"] == 0 + assert (summary["input_tokens"], summary["output_tokens"]) == (200, 40) + assert summary["models"] == ("claude-sonnet-4-5",) + assert summary["duration_ms"] == 1000 + assert summary["start_time"].startswith("2026-09-30T") + + +def test_error_count_counts_error_spans(): + rows = _deep_agent_rows() + rows[2]["status"] = "STATUS_CODE_ERROR" + trace = trace_from_rows("t1", rows) + assert trace is not None + assert trace["summary"]["error_count"] == 1 + assert trace["summary"]["status"] == "ok" # root span status; the UI uses error_count for "failed" + assert trace["spans"][2]["status"] == "error" + + +def test_offsets_are_relative_to_trace_start_in_ms(): + trace = trace_from_rows("t1", _deep_agent_rows()) + assert trace is not None + spans = {s["span_id"]: s for s in trace["spans"]} + assert spans["root"]["start_offset_ms"] == 0 + assert spans["task"]["start_offset_ms"] == 200 + assert spans["task"]["duration_ms"] == 700 + assert spans["root"]["parent_span_id"] is None + assert spans["task"]["parent_span_id"] == "root" + + +def test_span_from_row_optional_fields(): + span = span_from_row(_row("s", "", "x", "chain", "a", status="STATUS_CODE_UNSET"), T0) + assert (span["model"], span["parent_span_id"], span["status"], span["litellm_request_id"]) == ( + None, + None, + "unset", + None, + ) + + +def test_agent_nodes_parent_and_per_agent_counts(): + trace = trace_from_rows("t1", _deep_agent_rows()) + assert trace is not None + assert trace["agents"] == ( + { + "name": "deep_research_agent", + "parent_agent": None, + "invocations": 1, + "llm_calls": 1, + "tool_calls": 1, + "duration_ms": 1000, + }, + { + "name": "researcher", + "parent_agent": "deep_research_agent", + "invocations": 1, + "llm_calls": 1, + "tool_calls": 1, + "duration_ms": 5, + }, + ) + + +def test_200_subagent_invocations_aggregate_into_one_node(): + trace = trace_from_rows("t1", _deep_agent_rows(researcher_invocations=200)) + assert trace is not None + assert [a["name"] for a in trace["agents"]] == ["deep_research_agent", "researcher"] + researcher = trace["agents"][1] + assert researcher["parent_agent"] == "deep_research_agent" + assert researcher["invocations"] == 200 + assert researcher["llm_calls"] == 200 + assert researcher["tool_calls"] == 200 + assert researcher["duration_ms"] == pytest.approx(1000) + assert trace["summary"]["agent_count"] == 2 + assert trace["summary"]["span_count"] == 3 + 4 * 200 + + +def test_parent_agent_skips_same_name_ancestors(): + """A recursive agent (researcher -> researcher) still reports the nearest *different* agent.""" + rows = [ + _row("root", "", "lead", "agent", "lead"), + _row("r1", "root", "researcher", "agent", "researcher"), + _row("r2", "r1", "researcher", "agent", "researcher"), + ] + spans = [span_from_row(r, T0) for r in rows] + nodes = {n["name"]: n for n in agent_nodes(spans)} + assert nodes["researcher"]["parent_agent"] == "lead" + assert nodes["researcher"]["invocations"] == 2 + + +def test_parent_agent_stops_at_cyclic_parents(): + rows = [ + _row("self", "self", "researcher", "agent", "researcher"), + _row("first", "second", "researcher", "agent", "researcher"), + _row("second", "first", "researcher", "agent", "researcher"), + ] + spans = [span_from_row(row, T0) for row in rows] + assert agent_nodes(spans)[0]["parent_agent"] is None + + +def test_agent_nodes_ignores_spans_of_unknown_agents(): + spans = [span_from_row(_row("t", "", "tool", "tool", "ghost"), T0)] + assert agent_nodes(spans) == () + + +# ---------------------------------------------------------------- list helpers + + +def test_cursor_round_trip(): + cursor = encode_cursor(1790742989377, "4bad42b84e9de3ba46fc870185f8f023") + assert decode_cursor(cursor) == (1790742989377, "4bad42b84e9de3ba46fc870185f8f023") + assert decode_cursor(None) == (0, "") + assert decode_cursor("") == (0, "") + + +@pytest.mark.parametrize("cursor", ["abc", "bm90LWpzb24=", "WzEsIDJd", "WzAsICJ0Il0="]) +def test_invalid_cursor_is_rejected(cursor): + with pytest.raises(ValueError, match="Invalid trace cursor"): + decode_cursor(cursor) + + +def test_trace_summary_from_row(): + summary = trace_summary_from_row( + { + "trace_id": "t1", + "name": "deep_research_agent", + "service": "agent-demo", + "input_preview": "hi", + "start_ms": 1790742989377, + "duration_ms": 51385, + "status": "STATUS_CODE_OK", + "span_count": "126", + "agent_count": "2", + "llm_calls": "7", + "tool_calls": "26", + "error_count": "1", + "input_tokens": "30175", + "output_tokens": "2620", + "models": ["claude-sonnet-4-5"], + } + ) + assert summary["status"] == "ok" + assert (summary["span_count"], summary["error_count"]) == (126, 1) + assert summary["start_time"] == "2026-09-30T04:36:29.377000+00:00" + + +@pytest.mark.asyncio +async def test_list_traces_sets_next_cursor_on_full_page(): + client = MagicMock() + row = { + "trace_id": "t2", + "trace_ref": "ref2", + "name": "a", + "service": "s", + "input_preview": "", + "start_ms": 1000, + "duration_ms": 1, + "status": "STATUS_CODE_OK", + "span_count": 1, + "agent_count": 1, + "llm_calls": 0, + "tool_calls": 0, + "error_count": 0, + "input_tokens": 0, + "output_tokens": 0, + "models": [], + } + client.query = AsyncMock(return_value=[row, {**row, "trace_id": "t1", "trace_ref": "ref1", "start_ms": 900}]) + store = ClickHouseTraceStore(client) + scope: TraceScope = {"team_ids": ("team-a",), "api_key_hash": ""} + + page = await store.list_traces(scope, 0, 2000, limit=2) + assert [t["trace_id"] for t in page["data"]] == ["t2", "t1"] + assert page["next_cursor"] is not None + assert decode_cursor(page["next_cursor"]) == (900, "ref1") + params = client.query.call_args.args[1] + assert params["team_ids"] == ("team-a",) and params["limit"] == 2 and params["cursor_ms"] == 0 + + page = await store.list_traces(scope, 0, 2000, cursor=page["next_cursor"], limit=3) + assert page["next_cursor"] is None + assert client.query.call_args.args[1]["cursor_trace_id"] == "ref1" + + +@pytest.mark.asyncio +async def test_get_span_not_found_and_found(): + client = MagicMock() + client.query = AsyncMock(return_value=[]) + store = ClickHouseTraceStore(client) + scope: TraceScope = {"team_ids": (), "api_key_hash": ""} + assert await store.get_span("t", "s", scope) is None + client.query = AsyncMock(return_value=[{"span_id": "s", "input": "i", "output": "o", "attributes": {"k": "v"}}]) + assert await store.get_span("t", "s", scope) == { + "span_id": "s", + "input": "i", + "output": "o", + "attributes": {"k": "v"}, + } diff --git a/tests/test_litellm_rust/test_traces.py b/tests/test_litellm_rust/test_traces.py index 21f61ad24bc..447ca2ce4bb 100644 --- a/tests/test_litellm_rust/test_traces.py +++ b/tests/test_litellm_rust/test_traces.py @@ -1,11 +1,12 @@ import base64 +import gzip import json from typing import Final from urllib.parse import parse_qs, urlsplit import pytest -from litellm.rust_bridge.traces import encode_rows, ensure_schema, query +from litellm.rust_bridge._native import NativeTraceStorage from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec pytestmark = pytest.mark.requires_rust_extension @@ -14,14 +15,9 @@ pytestmark = pytest.mark.requires_rust_extension @pytest.mark.asyncio async def test_trace_reader_projects_connection_and_parameters(recording_server: RecordingServer) -> None: recording_server.enqueue(ResponseSpec(body={"data": [{"trace_id": "trace-1"}]})) - rows: Final = await query( - recording_server.base_url + "?database=wrong&user=wrong&password=wrong", - "trace_test", - "reader", - "p@ss/word%", - "SELECT {trace_id:String} AS trace_id", - {"trace_id": "trace-1"}, - ) + reader_url: Final = recording_server.base_url.replace("http://", "http://reader:p%40ss%2Fword%25@") + storage: Final = NativeTraceStorage("trace_test", recording_server.base_url, reader_url + "?database=wrong") + rows: Final = json.loads(await storage.query("SELECT {trace_id:String} AS trace_id", {"trace_id": "trace-1"})) request: Final = recording_server.requests[0] parameters: Final = parse_qs(urlsplit(request.path).query) assert rows == [{"trace_id": "trace-1"}] @@ -37,15 +33,22 @@ async def test_trace_reader_projects_connection_and_parameters(recording_server: @pytest.mark.asyncio async def test_trace_reader_rejects_success_status_with_embedded_error(recording_server: RecordingServer) -> None: recording_server.enqueue(ResponseSpec(body={"data": [], "exception": "query failed"})) + storage: Final = NativeTraceStorage("trace_test", recording_server.base_url, recording_server.base_url) with pytest.raises(RuntimeError, match="invalid or failed JSON"): - await query(recording_server.base_url, "trace_test", "reader", "password", "SELECT 1", {}) + await storage.query("SELECT 1", {}) @pytest.mark.asyncio -@pytest.mark.parametrize("database,retention", [("db; DROP DATABASE default", 7), ("traces", 0)]) -async def test_schema_binding_preserves_configuration_validation(database: str, retention: int) -> None: +async def test_schema_binding_rejects_invalid_database() -> None: with pytest.raises(ValueError, match=r"database.*retention"): - await ensure_schema("http://localhost:8123", database, "writer", "password", retention, 14) + NativeTraceStorage("db; DROP DATABASE default", "http://localhost:8123") + + +@pytest.mark.asyncio +async def test_schema_binding_rejects_non_positive_retention() -> None: + storage: Final = NativeTraceStorage("traces", "http://localhost:8123") + with pytest.raises(ValueError, match=r"database.*retention"): + await storage.ensure_schema(0, 14) @pytest.mark.asyncio @@ -53,15 +56,10 @@ async def test_schema_setup_uses_writer_credentials_and_rejects_failed_statement recording_server.expected_requests = 2 recording_server.enqueue(ResponseSpec(body="")) recording_server.enqueue(ResponseSpec(status=403, body="denied")) + writer_url: Final = recording_server.base_url.replace("http://", "http://writer:p%40ss%2Fword%25@") + storage: Final = NativeTraceStorage("trace_test", writer_url + "?database=wrong&readonly=1") with pytest.raises(RuntimeError, match="schema setup failed with HTTP status 403"): - await ensure_schema( - recording_server.base_url + "?database=wrong&readonly=1", - "trace_test", - "writer", - "p@ss/word%", - 7, - 14, - ) + await storage.ensure_schema(7, 14) assert len(recording_server.requests) == 2 assert recording_server.requests[0].raw_body.startswith(b"CREATE DATABASE IF NOT EXISTS") assert recording_server.requests[1].raw_body.startswith(b"CREATE TABLE IF NOT EXISTS") @@ -71,8 +69,15 @@ async def test_schema_setup_uses_writer_credentials_and_rejects_failed_statement ).decode() -def test_insert_encoding_preserves_nanoseconds_through_bridge() -> None: - assert json.loads(encode_rows([{"Timestamp": 1_234_567_890, "Input": "hello"}])) == { +@pytest.mark.asyncio +async def test_insert_encodes_and_sends_rows(recording_server: RecordingServer) -> None: + recording_server.enqueue(ResponseSpec(body="")) + storage: Final = NativeTraceStorage("trace_test", recording_server.base_url) + await storage.insert_rows("otel_traces", [{"Timestamp": 1_234_567_890, "Input": "hello"}]) + request: Final = recording_server.requests[0] + assert json.loads(gzip.decompress(request.raw_body)) == { "Input": "hello", "Timestamp": "1970-01-01T00:00:01.23456789Z", } + assert parse_qs(urlsplit(request.path).query)["query"] == ["INSERT INTO `trace_test`.otel_traces FORMAT JSONEachRow"] + assert request.headers["content-encoding"] == "gzip" diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 03e29c02021..1cb1951d96a 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -25,7 +25,7 @@ "jwt-decode": "4.0.0", "lucide-react": "0.513.0", "moment": "2.31.0", - "next": "16.3.3", + "next": "16.3.6", "next-themes": "^0.4.6", "nuqs": "^2.9.4", "openai": "4.104.0", @@ -2061,9 +2061,9 @@ } }, "node_modules/@next/env": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.3.tgz", - "integrity": "sha512-U2eYQRwXj+dsqxV79zFqExDdatnNY/ZWc2nsJU1p/OgT7fd3dXwlF6OjYaFQCfMoeTA19PWq+wVmYgimVA+V+g==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.6.tgz", + "integrity": "sha512-x9Vblze1EbtltQYnNH38xCPWU3TVfBd1eXqA3+w9+BTpedkkdNpAaltXlGQ/nsc1+E0mVTNrtcbX3GoO09zeLQ==", "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { @@ -2078,9 +2078,9 @@ } }, "node_modules/@next/swc-darwin-arm64": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.3.3.tgz", - "integrity": "sha512-8Hiv32QJPwdV6KYJ8meR9SBA061tQqnIKTJDocvOXlEQqib0xMFpzArosuffFUUc0sslbh7QQ8a3Yey1QV8EIw==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.3.6.tgz", + "integrity": "sha512-E/7GEqaUkt8mk/T8v9lAnrhzR06kdq1ZBkC12F8tAMkdIadwNp3H1KqHynDHrpcTlGCUdq/qu6vUL2aYVyYBdw==", "cpu": [ "arm64" ], @@ -2094,9 +2094,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.3.3.tgz", - "integrity": "sha512-A1lgKgwVchRYmSe467zdwhxT9040dd8lH+o65sL5Jet8fjB4kegw/rDyPIpYVRb6jAqwXFOJpjIXJLxQKLiE3A==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.3.6.tgz", + "integrity": "sha512-yBE893/nDWTlaiBD1p+qgt7NUen4U5R6FXyH0s67Npq1S3E0cVSef1WIXC2xBRgQvwAvJq6DnS6Y6PrY0cy4Ew==", "cpu": [ "x64" ], @@ -2110,9 +2110,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.3.3.tgz", - "integrity": "sha512-bf0FIssMFueU2dm7vQEWWxk0c8UjKTdW0yzuh0sQsD8pf1+KCLDdaqhYZNMYGmXwEOiHAUzgBKudovIlcvvBjg==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.3.6.tgz", + "integrity": "sha512-KJDpjBqBPYlvkivmyrp+Qys6k/7ksbqGQvRVc6ZEGfR+cjQxx+nUkJaWmNZJsmoOrqYNbaXByF8wa0lBwDhB3Q==", "cpu": [ "arm64" ], @@ -2129,9 +2129,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.3.3.tgz", - "integrity": "sha512-W7viwCk9JY/cAkdz/A273rd5bb3RgT/IHwR7Upv90tunjBWNtAAhGhoecHh+teRNRSinuAFmE+l7fwZ4YKkrXg==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.3.6.tgz", + "integrity": "sha512-mqNg2K+hvWskSRb/QM+Ix412DvBsuSF0XV+frTSw5vmoucNnIlynFwKYew8D01bfATErMOM7Bujrf0BA5DRKFA==", "cpu": [ "arm64" ], @@ -2148,9 +2148,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.3.3.tgz", - "integrity": "sha512-0W46zw1N3ODpI6n0GeivHvvob1pooozgZVqy65k0mh4/7vr+FbY9+WpHzNVXjHipJf/A3FDheBG19H1s5A25rA==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.3.6.tgz", + "integrity": "sha512-nFncBNGAYouRHjRVaITs9beZRfhX4ssVwpnvPIAbkZVH6LtGoAVlH4bJ8Cnf9SOo9bsXgPFer/GdHtEE3JNOkw==", "cpu": [ "x64" ], @@ -2167,9 +2167,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.3.3.tgz", - "integrity": "sha512-H4mBso8ZTMBPtdT0PN0pBx2ayTvQuTuvS6qT13d77yVFJXAPCxkyIhLTmdMaGTJs0krQYI/qpzdHijCeihXhbg==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.3.6.tgz", + "integrity": "sha512-5Mf3cHDGR/Iz0ng2Bj3zUR3p5QS9YK3Hn2QiAfavFmyF48zwThAjpFoiTKNIcOHLYS4zEk+gzyJ/9deQ2ZB8yQ==", "cpu": [ "x64" ], @@ -2186,9 +2186,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.3.3.tgz", - "integrity": "sha512-cTMUJpcEGmeywofCUfhR+rSsoE33+rVPnPEYNTNdLNlsOeEg/vktOsKUSTb28vUGqD2jkm4Zaskcwn7OCI6FQg==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.3.6.tgz", + "integrity": "sha512-0jkJy0C2kbrJWTk4YLa3xk80pVBpx8FCHJym7CnUfDAXe/FWv5qT7SQJbR0KuemyxaEDlEx5WT4VQJoTW+/9Qw==", "cpu": [ "arm64" ], @@ -2202,9 +2202,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.3.3.tgz", - "integrity": "sha512-2VR4cTBzHXaBjnGsuH6GyJjENzQOmHeAh11uY1iUhjm3j5dEUrVJuUj+VL78jaGi/Dik8xS76zEj18BsFhlVZQ==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.3.6.tgz", + "integrity": "sha512-/YXjI1e5OXcZ7YpxRwgP/1jAV/SBKTzeVKqN2mk7mLpcICsyn3Gl5+dIfDTJp70M0ccMhyMMRso4v6mPDCGepg==", "cpu": [ "x64" ], @@ -9712,12 +9712,12 @@ "license": "MIT" }, "node_modules/next": { - "version": "16.3.3", - "resolved": "https://registry.npmjs.org/next/-/next-16.3.3.tgz", - "integrity": "sha512-tuRTx1nQ/yVw83cwJBo9F+njGUgMn3UHQycreWHB8XsStvvAh1AthbI8/4IpKnFaF58F+iSiHejYOlMQ/eq83g==", + "version": "16.3.6", + "resolved": "https://registry.npmjs.org/next/-/next-16.3.6.tgz", + "integrity": "sha512-L+otWM/aQbYTx98aZhgEoMb4bZAXx1YVW4UMA/vuCyCoWG5HJyZUili8QAkqzrcC+5///tsz3s0M+SlyB5bLMw==", "license": "MIT", "dependencies": { - "@next/env": "16.3.3", + "@next/env": "16.3.6", "@swc/helpers": "0.5.23", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", @@ -9731,15 +9731,15 @@ "node": ">=20.9.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "16.3.3", - "@next/swc-darwin-x64": "16.3.3", - "@next/swc-linux-arm64-gnu": "16.3.3", - "@next/swc-linux-arm64-musl": "16.3.3", - "@next/swc-linux-x64-gnu": "16.3.3", - "@next/swc-linux-x64-musl": "16.3.3", - "@next/swc-win32-arm64-msvc": "16.3.3", - "@next/swc-win32-x64-msvc": "16.3.3", - "sharp": "^0.35.3" + "@next/swc-darwin-arm64": "16.3.6", + "@next/swc-darwin-x64": "16.3.6", + "@next/swc-linux-arm64-gnu": "16.3.6", + "@next/swc-linux-arm64-musl": "16.3.6", + "@next/swc-linux-x64-gnu": "16.3.6", + "@next/swc-linux-x64-musl": "16.3.6", + "@next/swc-win32-arm64-msvc": "16.3.6", + "@next/swc-win32-x64-msvc": "16.3.6", + "sharp": "^0.35.4" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 3bf32d37faf..0830e233bbe 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -41,7 +41,7 @@ "jwt-decode": "4.0.0", "lucide-react": "0.513.0", "moment": "2.31.0", - "next": "16.3.3", + "next": "16.3.6", "next-themes": "^0.4.6", "nuqs": "^2.9.4", "openai": "4.104.0", diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index d0a5349364f..7ffe0e0ec85 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2121,12 +2121,21 @@ export const agentTraceListCall = async ({ return apiClient.get(`/v1/traces`, { accessToken, query }); }; -export const agentTraceCall = async (accessToken: string, traceId: string): Promise => - apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}`, { accessToken }); +export const agentTraceCall = async (accessToken: string, traceId: string, traceRef?: string): Promise => + apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}`, { + accessToken, + query: { trace_ref: traceRef || undefined }, + }); -export const agentTraceSpanCall = async (accessToken: string, traceId: string, spanId: string): Promise => +export const agentTraceSpanCall = async ( + accessToken: string, + traceId: string, + spanId: string, + traceRef?: string, +): Promise => apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}`, { accessToken, + query: { trace_ref: traceRef || undefined }, }); export const adminSpendLogsCall = async (accessToken: string) => { diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx index 2bfa3c320a7..61e4c737b24 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx @@ -67,7 +67,7 @@ export function AgentTracesSection({ timeControls, onRunOpenChange, }: AgentTracesSectionProps) { - const [openTraceId, setOpenTraceId] = useState(null); + const [openTrace, setOpenTrace] = useState(null); const [query, setQuery] = useState(""); const [service, setService] = useState(ALL_SERVICES); const [status, setStatus] = useState("all"); @@ -96,9 +96,9 @@ export function AgentTracesSection({ apply(hours); }; - const openRun = (traceId: string | null) => { - setOpenTraceId(traceId); - onRunOpenChange?.(traceId !== null); + const openRun = (trace: TraceSummary | null) => { + setOpenTrace(trace); + onRunOpenChange?.(trace !== null); }; if (traces.notEnabledDetail !== null) return ; @@ -120,8 +120,15 @@ export function AgentTracesSection({ ); } - if (openTraceId !== null) { - return openRun(null)} />; + if (openTrace !== null) { + return ( + openRun(null)} + /> + ); } return ( diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx index e4faff1621a..d3912b7be5c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx @@ -14,7 +14,7 @@ interface AgentTracesTableProps { error: Error | null; hasMore: boolean; onLoadMore: () => void; - onOpenTrace: (traceId: string) => void; + onOpenTrace: (trace: TraceSummary) => void; } /** Spend is only on summaries once the spend-enrichment PR lands; show Cost when it's there. */ @@ -82,9 +82,9 @@ export function AgentTracesTable({ {traces.map((run) => ( onOpenTrace(run.trace_id)} + onClick={() => onOpenTrace(run)} className="h-9 cursor-pointer border-b border-border/60 text-[12px] hover:bg-accent/50" > const errorReason = (headline: string): string => /^([A-Za-z_][\w.]*)\(/.exec(headline)?.[1] ?? "error"; /** Shared lazy fetch of one span's full input / output / attributes. */ -export function useSpanDetail(accessToken: string, traceId: string, spanId: string | null) { +export function useSpanDetail(accessToken: string, traceId: string, spanId: string | null, traceRef?: string) { const queryOptions: UseQueryOptions = { - queryKey: ["agentTraceSpan", traceId, spanId, accessToken], - queryFn: () => agentTraceSpanCall(accessToken, traceId, spanId as string), + queryKey: ["agentTraceSpan", traceId, traceRef, spanId, accessToken], + queryFn: () => agentTraceSpanCall(accessToken, traceId, spanId as string, traceRef), enabled: spanId !== null, staleTime: Infinity, }; @@ -121,12 +121,13 @@ function Payload({ label, value, mono }: { label: string; value: string; mono: b interface DetailContentProps { accessToken: string; traceId: string; + traceRef?: string; span: Span; } /** Content tab: the error first (if any), then what went in and what came out. */ -export function DetailContent({ accessToken, traceId, span }: DetailContentProps) { - const detailQuery = useSpanDetail(accessToken, traceId, span.span_id); +export function DetailContent({ accessToken, traceId, traceRef, span }: DetailContentProps) { + const detailQuery = useSpanDetail(accessToken, traceId, span.span_id, traceRef); const detail = detailQuery.data; const isTool = span.type === "tool"; const empty = detail && !detail.input && !detail.output; diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.test.tsx index a15584e6d41..9a541d26966 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.test.tsx @@ -125,7 +125,7 @@ describe("DetailPane", () => { expect(screen.getByRole("tab", { name: "Attributes" })).toBeInTheDocument(); expect(await screen.findByText("You are a LiteLLM support agent.")).toBeInTheDocument(); expect(screen.getByText("get_customer_plan")).toBeInTheDocument(); - expect(vi.mocked(agentTraceSpanCall)).toHaveBeenCalledWith("sk-test", "t1", "llm1"); + expect(vi.mocked(agentTraceSpanCall)).toHaveBeenCalledWith("sk-test", "t1", "llm1", undefined); }); it("shows a tool failure as 'Tool · ' with the exception line and no traceback", async () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx index 0a4dd3c5029..4424b95cd52 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx @@ -67,7 +67,12 @@ function SpanPane({ }) { const [tab, setTab] = useState("content"); const traceId = trace.summary.trace_id; - const detailQuery = useSpanDetail(accessToken, traceId, tab === "attributes" ? span.span_id : null); + const detailQuery = useSpanDetail( + accessToken, + traceId, + tab === "attributes" ? span.span_id : null, + trace.summary.trace_ref, + ); const tokens = span.input_tokens + span.output_tokens; return (