From 26a9b02f7ba19031cc9e1a20eb2dbc2b8d5873e4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:46:50 -0500 Subject: [PATCH 1/5] feat(otel): let team and key Arize callbacks choose the OTLP transport (#44492) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/arize/arize.py | 18 ++- .../integrations/otel/presets/destinations.py | 2 +- .../initialize_dynamic_callback_params.py | 12 +- litellm/proxy/_types.py | 3 + .../callback_config_validation.py | 42 +++++- .../team_callback_endpoints.py | 2 +- litellm/types/utils.py | 4 + .../otel/test_otel_v2_destinations.py | 120 ++++++++++++++++ .../test_callback_config_validation.py | 131 ++++++++++++++++++ .../test_callback_management_endpoints.py | 64 +++++++++ .../proxy/test_credential_slot_registry.py | 1 + .../unit/proxy/test_litellm_pre_call_utils.py | 35 +++++ .../src/components/callback_info_helpers.tsx | 6 + .../components/team/LoggingSettings.test.tsx | 88 ++++++++++++ .../src/components/team/LoggingSettings.tsx | 53 ++++--- 15 files changed, 555 insertions(+), 26 deletions(-) diff --git a/litellm/integrations/arize/arize.py b/litellm/integrations/arize/arize.py index 58024cb2e1e..24279b443aa 100644 --- a/litellm/integrations/arize/arize.py +++ b/litellm/integrations/arize/arize.py @@ -17,7 +17,7 @@ from litellm.integrations.arize._utils import ArizeOTELAttributes from litellm.integrations.opentelemetry import _MAX_DYNAMIC_TRACER_PROVIDERS, OpenTelemetry, OpenTelemetryConfig from litellm.types.integrations.arize import ArizeConfig from litellm.types.services import ServiceLoggerPayload -from litellm.types.utils import StandardCallbackDynamicParams +from litellm.types.utils import ArizeOtlpProtocol, StandardCallbackDynamicParams if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -174,7 +174,7 @@ class ArizeLogger(OpenTelemetry): _utils.set_attributes(span, kwargs, response_obj, ArizeOTELAttributes) @staticmethod - def get_arize_config() -> ArizeConfig: + def get_arize_config(otlp_protocol: ArizeOtlpProtocol | None = None) -> ArizeConfig: """ Helper function to get Arize configuration. @@ -195,7 +195,19 @@ class ArizeLogger(OpenTelemetry): endpoint = None protocol: Protocol = "otlp_grpc" - if grpc_endpoint: + if otlp_protocol == "http/protobuf": + protocol = "otlp_http" + if http_endpoint: + endpoint = http_endpoint + elif grpc_endpoint: + base: Final = grpc_endpoint.rstrip("/") + endpoint = f"{base}/traces" if base.endswith("/v1") else base + else: + endpoint = "https://otlp.arize.com/v1/traces" + elif otlp_protocol == "grpc": + protocol = "otlp_grpc" + endpoint = grpc_endpoint or http_endpoint or "https://otlp.arize.com/v1" + elif grpc_endpoint: protocol = "otlp_grpc" endpoint = grpc_endpoint elif http_endpoint: diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py index 67450684658..91341a57634 100644 --- a/litellm/integrations/otel/presets/destinations.py +++ b/litellm/integrations/otel/presets/destinations.py @@ -63,7 +63,7 @@ def _langfuse_destination(params: StandardCallbackDynamicParams) -> "_Destinatio def _arize_destination(params: StandardCallbackDynamicParams) -> "_Destination | None": from litellm.integrations.arize.arize import ArizeLogger - config: Final = ArizeLogger.get_arize_config() + config: Final = ArizeLogger.get_arize_config(params.get("arize_otlp_protocol")) return (config.endpoint, config.protocol) diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index 9ac78e3b917..d2ba17e169a 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -4,7 +4,12 @@ from contextlib import contextmanager from contextvars import ContextVar from typing import Any, Final -from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams +from litellm.types.utils import ( + ARIZE_OTLP_PROTOCOLS, + OTEL_SPAN_SCOPES, + TRUSTED_CALLBACK_VARS_FIELD, + StandardCallbackDynamicParams, +) _CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata") _inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar( @@ -81,6 +86,11 @@ def validate_langfuse_span_scope_value(value: str) -> None: raise ValueError(f"Invalid langfuse_span_scope {value!r}: must be one of {sorted(OTEL_SPAN_SCOPES)}") +def validate_arize_otlp_protocol_value(value: str) -> None: + if value not in ARIZE_OTLP_PROTOCOLS: + raise ValueError(f"Invalid arize_otlp_protocol {value!r}: must be one of {sorted(ARIZE_OTLP_PROTOCOLS)}") + + # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict _supported_callback_params: Final[tuple[str, ...]] = ( "langfuse_public_key", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 291e991f6bd..c1a97aa73c4 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -25,6 +25,7 @@ from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from litellm._uuid import uuid from litellm.constants import DEFAULT_STAGGER_WINDOW_SECONDS, MCP_STDIO_ALLOWED_COMMANDS from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + validate_arize_otlp_protocol_value, validate_langfuse_environment_value, validate_langfuse_span_scope_value, validate_no_callback_env_reference, @@ -2383,6 +2384,8 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): validate_langfuse_environment_value(callback_vars[key]) if key == "langfuse_span_scope": validate_langfuse_span_scope_value(callback_vars[key]) + if key == "arize_otlp_protocol": + validate_arize_otlp_protocol_value(callback_vars[key]) return values diff --git a/litellm/proxy/common_utils/callback_config_validation.py b/litellm/proxy/common_utils/callback_config_validation.py index 6a0fcbe0bb3..5b9fb2e3977 100644 --- a/litellm/proxy/common_utils/callback_config_validation.py +++ b/litellm/proxy/common_utils/callback_config_validation.py @@ -15,15 +15,20 @@ _NEWRELIC_VAR_PREFIX: Final = "newrelic_" _LANGFUSE_OTEL_CALLBACK: Final = "langfuse_otel" _LANGFUSE_SPAN_SCOPE_VAR: Final = "langfuse_span_scope" _ARIZE_CALLBACK: Final = "arize" +_ARIZE_OTLP_PROTOCOL_VAR: Final = "arize_otlp_protocol" _ARIZE_SAMPLING_RATE_VARS: Final[frozenset[str]] = frozenset( {"arize_success_sampling_rate", "arize_error_sampling_rate"} ) -def callback_config_error(callback_name: str | None, callback_vars: Mapping[str, str] | None) -> str | None: +def callback_config_error( + callback_name: str | None, callback_vars: Mapping[str, str] | None, callback_type: str | None = None +) -> str | None: if not callback_vars: return None - arize_error: Final = _arize_sampling_rate_error(callback_name, callback_vars) + arize_error: Final = _arize_sampling_rate_error(callback_name, callback_vars) or _arize_otlp_protocol_error( + callback_name, callback_vars, callback_type + ) if arize_error is not None: return arize_error langfuse_error: Final = _langfuse_environment_error(callback_vars) or _langfuse_span_scope_error( @@ -94,7 +99,9 @@ _VAR_FAMILIES: Final[Mapping[str, str]] = MappingProxyType( } ) -_FAMILY_OPTION_VARS: Final[frozenset[str]] = frozenset({_LANGFUSE_SPAN_SCOPE_VAR, *_ARIZE_SAMPLING_RATE_VARS}) +_FAMILY_OPTION_VARS: Final[frozenset[str]] = frozenset( + {_LANGFUSE_SPAN_SCOPE_VAR, _ARIZE_OTLP_PROTOCOL_VAR, *_ARIZE_SAMPLING_RATE_VARS} +) def _family_of(var: str) -> str | None: @@ -217,7 +224,10 @@ def _logging_entry_error(entry: object) -> str | None: callback_name: Final = entry.get("callback_name") if not isinstance(callback_name, str) or not isinstance(entry.get("callback_vars"), Mapping): return None - return callback_config_error(callback_name, _entry_callback_vars(entry)) + callback_type_raw: Final = entry.get("callback_type") + return callback_config_error( + callback_name, _entry_callback_vars(entry), callback_type_raw if isinstance(callback_type_raw, str) else None + ) def _arize_sampling_rate_error(callback_name: str | None, callback_vars: Mapping[str, str]) -> str | None: @@ -236,6 +246,30 @@ def _arize_sampling_rate_error(callback_name: str | None, callback_vars: Mapping return None +def _arize_otlp_protocol_error( + callback_name: str | None, callback_vars: Mapping[str, str], callback_type: str | None = None +) -> str | None: + value: Final = callback_vars.get(_ARIZE_OTLP_PROTOCOL_VAR) + if value is None: + return None + if callback_name != _ARIZE_CALLBACK: + return f"{_ARIZE_OTLP_PROTOCOL_VAR} applies to the {_ARIZE_CALLBACK} callback only, not {callback_name!r}" + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + validate_arize_otlp_protocol_value, + ) + + try: + validate_arize_otlp_protocol_value(value) + except ValueError as e: + return str(e) + if callback_type == "failure": + return f"{_ARIZE_OTLP_PROTOCOL_VAR} needs callback_type 'success' or 'success_and_failure'; failure-only Arize callbacks export over the proxy's own Arize transport" + if not is_otel_v2_enabled(): + return "Per-team Arize transport selection requires the proxy to run with LITELLM_OTEL_V2=true." + return None + + def _newrelic_config_error(callback_vars: Mapping[str, str]) -> str | None: """Per-team New Relic routing runs on the OTel v2 path only. diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index a59342fa044..471a0814ae5 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -61,7 +61,7 @@ def _callback_config_error(message: str) -> HTTPException: def _validate_team_callback(data: "AddTeamCallback") -> None: - error: Final = callback_config_error(data.callback_name, data.callback_vars) + error: Final = callback_config_error(data.callback_name, data.callback_vars, data.callback_type) if error is not None: raise _callback_config_error(error) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d3fe65037e2..bd7ccc5944f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3648,6 +3648,9 @@ OPENAI_RESPONSE_HEADERS: Final = [ OtelSpanScope = Literal["full", "llm_only"] OTEL_SPAN_SCOPES: Final[frozenset[str]] = frozenset(get_args(OtelSpanScope)) +ArizeOtlpProtocol = Literal["grpc", "http/protobuf"] +ARIZE_OTLP_PROTOCOLS: Final[frozenset[str]] = frozenset(get_args(ArizeOtlpProtocol)) + class StandardCallbackDynamicParams(TypedDict, total=False): # Langfuse dynamic params @@ -3681,6 +3684,7 @@ class StandardCallbackDynamicParams(TypedDict, total=False): arize_space_id: str | None arize_success_sampling_rate: ReadOnly[float | None] arize_error_sampling_rate: ReadOnly[float | None] + arize_otlp_protocol: ReadOnly[ArizeOtlpProtocol | None] # PostHog dynamic params posthog_api_key: str | None diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index b80d75adc0b..2d7e0745039 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -3081,6 +3081,126 @@ class TestBackendEndpointParity: assert destination.endpoint == "https://trace.wandb.ai/otel/v1/traces" +class TestArizeOtlpProtocol: + @staticmethod + def _arize(monkeypatch, endpoint_env, **env): + monkeypatch.delenv("ARIZE_ENDPOINT", raising=False) + monkeypatch.delenv("ARIZE_HTTP_ENDPOINT", raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + return destination_for("arize", {"arize_space_id": "s", "arize_api_key": "k", **endpoint_env}) + + def test_http_transport_rewrites_the_grpc_endpoint_to_the_traces_path(self, monkeypatch): + destination = self._arize( + monkeypatch, + {"arize_otlp_protocol": "http/protobuf"}, + ARIZE_ENDPOINT="https://arize.internal.example/v1", + ) + + assert destination.protocol == "otlp_http" + assert destination.endpoint == "https://arize.internal.example/v1/traces" + + def test_http_transport_prefers_the_http_endpoint(self, monkeypatch): + destination = self._arize( + monkeypatch, + {"arize_otlp_protocol": "http/protobuf"}, + ARIZE_ENDPOINT="https://arize.internal.example/v1", + ARIZE_HTTP_ENDPOINT="https://http.arize.internal.example/v1/traces", + ) + + assert destination.protocol == "otlp_http" + assert destination.endpoint == "https://http.arize.internal.example/v1/traces" + + def test_http_transport_falls_back_to_the_arize_cloud_http_endpoint(self, monkeypatch): + destination = self._arize(monkeypatch, {"arize_otlp_protocol": "http/protobuf"}) + + assert destination.protocol == "otlp_http" + assert destination.endpoint == "https://otlp.arize.com/v1/traces" + + def test_grpc_transport_accepts_the_http_endpoint_as_the_grpc_target(self, monkeypatch): + destination = self._arize( + monkeypatch, + {"arize_otlp_protocol": "grpc"}, + ARIZE_HTTP_ENDPOINT="https://http.arize.internal.example/v1/traces", + ) + + assert destination.protocol == "otlp_grpc" + assert destination.endpoint == "https://http.arize.internal.example/v1/traces" + + def test_no_transport_var_keeps_the_grpc_env_default(self, monkeypatch): + destination = self._arize(monkeypatch, {}, ARIZE_ENDPOINT="https://arize.internal.example") + + assert destination.protocol == "otlp_grpc" + assert destination.endpoint == "https://arize.internal.example" + + def test_the_http_destination_builds_an_http_exporter_on_the_traces_path(self, monkeypatch): + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter as HTTPSpanExporter, + ) + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + from litellm.integrations.otel.plumbing.providers import _destination_processor + + destination = self._arize( + monkeypatch, + {"arize_otlp_protocol": "http/protobuf"}, + ARIZE_ENDPOINT="https://arize.internal.example/v1", + ) + processor = _destination_processor(destination) + try: + assert isinstance(processor, BatchSpanProcessor) + assert isinstance(processor.span_exporter, HTTPSpanExporter) + assert processor.span_exporter._endpoint == "https://arize.internal.example/v1/traces" + finally: + processor.shutdown() + + def test_the_grpc_destination_builds_a_grpc_exporter(self, monkeypatch): + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter as GRPCSpanExporter, + ) + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + from litellm.integrations.otel.plumbing.providers import _destination_processor + + destination = self._arize( + monkeypatch, + {"arize_otlp_protocol": "grpc"}, + ARIZE_ENDPOINT="https://arize.internal.example", + ) + processor = _destination_processor(destination) + try: + assert isinstance(processor, BatchSpanProcessor) + assert isinstance(processor.span_exporter, GRPCSpanExporter) + finally: + processor.shutdown() + + def test_a_team_metadata_protocol_reaches_the_destination(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("ARIZE_ENDPOINT", "https://arize.internal.example/v1") + monkeypatch.delenv("ARIZE_HTTP_ENDPOINT", raising=False) + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": { + "arize_space_id": "s", + "arize_api_key": "k", + "arize_otlp_protocol": "http/protobuf", + }, + } + ] + } + ) + + destinations = resolve_tenant_otel_destinations(auth) + + assert [d.protocol for d in destinations] == ["otlp_http"] + assert destinations[0].endpoint == "https://arize.internal.example/v1/traces" + + class TestIncompleteCredentials: """Half a credential set builds a non-empty but unusable header dict. Accepting it would suppress the operator's exporter and send the trace where it cannot land.""" diff --git a/tests/unit/proxy/common_utils/test_callback_config_validation.py b/tests/unit/proxy/common_utils/test_callback_config_validation.py index 33ae986a511..d10a96e2aef 100644 --- a/tests/unit/proxy/common_utils/test_callback_config_validation.py +++ b/tests/unit/proxy/common_utils/test_callback_config_validation.py @@ -121,6 +121,137 @@ def test_arize_sampling_rate_rejected_on_non_arize_callback(): assert callback_config_error("arize", {"arize_success_sampling_rate": "0.5"}) is None +class TestArizeOtlpProtocol: + def test_protocol_on_a_non_arize_callback_is_rejected(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + for callback_name in ["langfuse", "langfuse_otel", "datadog", None]: + error = callback_config_error(callback_name, {"arize_otlp_protocol": "grpc"}) + assert error is not None and "applies to the arize callback only" in error + finally: + is_otel_v2_enabled.cache_clear() + + def test_unknown_protocols_are_rejected(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + for bad in ["otlp_http", "HTTP/PROTOBUF", "http_json", "", "None"]: + error = callback_config_error("arize", {"arize_otlp_protocol": bad}) + assert error is not None and "arize_otlp_protocol" in error and "http/protobuf" in error + finally: + is_otel_v2_enabled.cache_clear() + + def test_protocol_is_rejected_while_otel_v2_is_off(self, monkeypatch): + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + error = callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}) + assert error is not None and "LITELLM_OTEL_V2" in error + finally: + is_otel_v2_enabled.cache_clear() + + def test_both_protocols_are_accepted_with_otel_v2_on(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + assert callback_config_error("arize", {"arize_otlp_protocol": "grpc"}) is None + assert callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}) is None + finally: + is_otel_v2_enabled.cache_clear() + + def test_protocol_is_not_a_family_credential(self): + stored = [{"arize_api_key": "k1"}] + assert cross_entry_family_error({"arize_otlp_protocol": "grpc"}, stored) is None + assert cross_entry_family_error({"arize_otlp_protocol": "http/protobuf"}, stored) is None + + def test_protocol_on_a_key_logging_entry_is_validated(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + invalid_metadata = { + "logging": [ + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"arize_otlp_protocol": "carrier-pigeon"}, + } + ] + } + error = logging_metadata_config_error(invalid_metadata) + assert error is not None and "arize_otlp_protocol" in error + + valid_metadata = { + "logging": [ + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"arize_otlp_protocol": "http/protobuf"}, + } + ] + } + assert logging_metadata_config_error(valid_metadata) is None + finally: + is_otel_v2_enabled.cache_clear() + + def test_failure_only_arize_callbacks_reject_the_protocol(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + error = callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}, "failure") + assert error is not None and "arize_otlp_protocol" in error and "failure" in error + assert callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}, "success") is None + assert ( + callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}, "success_and_failure") is None + ) + assert callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}) is None + finally: + is_otel_v2_enabled.cache_clear() + + def test_failure_only_key_logging_entry_rejects_the_protocol(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + failure_metadata = { + "logging": [ + { + "callback_name": "arize", + "callback_type": "failure", + "callback_vars": {"arize_otlp_protocol": "http/protobuf"}, + } + ] + } + error = logging_metadata_config_error(failure_metadata) + assert error is not None and "arize_otlp_protocol" in error + + success_metadata = { + "logging": [ + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"arize_otlp_protocol": "http/protobuf"}, + } + ] + } + assert logging_metadata_config_error(success_metadata) is None + finally: + is_otel_v2_enabled.cache_clear() + + def test_arize_sampling_rates_are_not_family_credentials(): """The rates choose what the Arize family exports, not where it sends, so an entry that repeats or adds a rate next to a stored Arize entry is not the diff --git a/tests/unit/proxy/management_endpoints/test_callback_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_callback_management_endpoints.py index b64877f7816..34fa164fd6d 100644 --- a/tests/unit/proxy/management_endpoints/test_callback_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_callback_management_endpoints.py @@ -361,6 +361,70 @@ class TestNewRelicTeamCallbackValidation: is_otel_v2_enabled.cache_clear() +class TestArizeOtlpProtocolValidation: + def test_valid_protocols_are_accepted(self): + from litellm.proxy._types import AddTeamCallback + + for protocol in ("grpc", "http/protobuf"): + data = AddTeamCallback( + callback_name="arize", + callback_type="success", + callback_vars={"arize_api_key": "k", "arize_space_id": "s", "arize_otlp_protocol": protocol}, + ) + assert data.callback_vars["arize_otlp_protocol"] == protocol + + def test_an_invalid_protocol_is_rejected_at_model_validation(self): + from pydantic import ValidationError + + from litellm.proxy._types import AddTeamCallback + + with pytest.raises(ValidationError, match="arize_otlp_protocol"): + AddTeamCallback( + callback_name="arize", + callback_type="success", + callback_vars={"arize_otlp_protocol": "otlp_grpc"}, + ) + + def test_a_failure_only_team_callback_rejects_the_protocol(self, monkeypatch): + from fastapi import HTTPException + + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.proxy._types import AddTeamCallback + from litellm.proxy.management_endpoints.team_callback_endpoints import _validate_team_callback + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + try: + with pytest.raises(HTTPException) as exc: + _validate_team_callback( + AddTeamCallback( + callback_name="arize", + callback_type="failure", + callback_vars={ + "arize_api_key": "k", + "arize_space_id": "s", + "arize_otlp_protocol": "http/protobuf", + }, + ) + ) + assert exc.value.status_code == 400 + assert "arize_otlp_protocol" in str(exc.value.detail) + finally: + is_otel_v2_enabled.cache_clear() + + def test_the_raw_otlp_env_var_name_stays_rejected(self): + from pydantic import ValidationError + + from litellm.proxy._types import AddTeamCallback + + with pytest.raises(ValidationError, match="Invalid callback variable"): + AddTeamCallback( + callback_name="arize", + callback_type="success", + callback_vars={"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf"}, + ) + + class TestNewRelicKeyLoggingValidation: """Key-level logging is written through key metadata, not /team/callback.""" diff --git a/tests/unit/proxy/test_credential_slot_registry.py b/tests/unit/proxy/test_credential_slot_registry.py index 02268dba361..044b00b33bb 100644 --- a/tests/unit/proxy/test_credential_slot_registry.py +++ b/tests/unit/proxy/test_credential_slot_registry.py @@ -89,6 +89,7 @@ CALLBACK_PARAM_CLASSIFICATION: Final[Mapping[str, Classification]] = MappingProx "arize_space_id": NotSecret("space identifier"), "arize_success_sampling_rate": NotSecret("sampling rate"), "arize_error_sampling_rate": NotSecret("sampling rate"), + "arize_otlp_protocol": NotSecret("transport protocol enum"), "posthog_api_key": Unplanted(), "posthog_api_url": NotSecret("sink endpoint URL"), "wandb_api_key": Unplanted(), diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index db2aec10fdb..e604604025a 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -8590,3 +8590,38 @@ def test_signoz_callback_vars_are_scoped_to_the_signoz_callback(): team_callback_settings_obj=None, ) assert under_other.callback_vars == {"langfuse_host": "https://cloud.langfuse.com"} + + +def test_arize_otlp_protocol_on_a_key_logging_entry_reaches_the_destination(monkeypatch): + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.proxy.litellm_pre_call_utils import resolve_tenant_otel_destinations + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("ARIZE_ENDPOINT", "https://arize.internal.example/v1") + monkeypatch.delenv("ARIZE_HTTP_ENDPOINT", raising=False) + is_otel_v2_enabled.cache_clear() + try: + auth = UserAPIKeyAuth( + api_key="hashed-key", + metadata={ + "logging": [ + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": { + "arize_space_id": "s", + "arize_api_key": "k", + "arize_otlp_protocol": "http/protobuf", + }, + } + ] + }, + team_metadata={}, + ) + + destinations = resolve_tenant_otel_destinations(auth) + + assert [d.protocol for d in destinations] == ["otlp_http"] + assert destinations[0].endpoint == "https://arize.internal.example/v1/traces" + finally: + is_otel_v2_enabled.cache_clear() diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx index 19020d92066..6ee092f030d 100644 --- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx @@ -20,6 +20,7 @@ interface CallbackConfig { supports_key_team_logging: boolean; dynamic_params: Record; dynamic_param_options?: Record; + success_event_params?: string[]; description: string; } @@ -34,7 +35,12 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ arize_space_id: "password", arize_success_sampling_rate: "number", arize_error_sampling_rate: "number", + arize_otlp_protocol: "select", }, + dynamic_param_options: { + arize_otlp_protocol: ["grpc", "http/protobuf"], + }, + success_event_params: ["arize_otlp_protocol"], description: "Arize Logging Integration", }, { diff --git a/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx b/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx index 5ce8ff71267..670bd710d38 100644 --- a/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/team/LoggingSettings.test.tsx @@ -239,6 +239,94 @@ describe("LoggingSettings", () => { ]); }); + it("offers the Arize OTLP protocol as a pick between grpc and http/protobuf rather than free text", async () => { + const user = userEvent.setup({ delay: null }); + const mockOnChange = vi.fn(); + const initialValue = [ + { + callback_name: "arize", + callback_type: "success", + callback_vars: {}, + }, + ]; + + renderWithProviders(); + + expect(screen.queryByPlaceholderText("os.environ/ARIZE_OTLP_PROTOCOL")).not.toBeInTheDocument(); + await user.click(screen.getByRole("combobox", { name: "arize otlp protocol" })); + expect((await screen.findAllByRole("option")).map((option) => option.textContent)).toEqual([ + "grpc", + "http/protobuf", + ]); + await user.click(screen.getByRole("option", { name: "http/protobuf" })); + + expect(mockOnChange).toHaveBeenCalledWith([ + expect.objectContaining({ callback_vars: expect.objectContaining({ arize_otlp_protocol: "http/protobuf" }) }), + ]); + }); + + it("hides the Arize OTLP protocol picker on a failure-only Arize callback", () => { + const mockOnChange = vi.fn(); + const initialValue = [ + { + callback_name: "arize", + callback_type: "failure", + callback_vars: {}, + }, + ]; + + renderWithProviders(); + + expect(screen.queryByRole("combobox", { name: "arize otlp protocol" })).not.toBeInTheDocument(); + expect(screen.getByPlaceholderText("os.environ/ARIZE_API_KEY")).toBeInTheDocument(); + expect(screen.getByPlaceholderText("os.environ/ARIZE_SPACE_ID")).toBeInTheDocument(); + }); + + it("drops the Arize OTLP protocol from callback_vars when the event type switches to failure", async () => { + const user = userEvent.setup({ delay: null }); + const mockOnChange = vi.fn(); + const initialValue = [ + { + callback_name: "arize", + callback_type: "success", + callback_vars: { arize_space_id: "s", arize_otlp_protocol: "http/protobuf" }, + }, + ]; + + renderWithProviders(); + + await user.click(screen.getByRole("combobox", { name: "Event Type" })); + await user.click(await screen.findByRole("option", { name: "Failure Only" })); + + const lastCall = mockOnChange.mock.calls[mockOnChange.mock.calls.length - 1]; + expect(lastCall[0][0].callback_type).toBe("failure"); + expect(lastCall[0][0].callback_vars).toEqual({ arize_space_id: "s" }); + }); + + it("keeps the Arize OTLP protocol in callback_vars when the event type switches to success_and_failure", async () => { + const user = userEvent.setup({ delay: null }); + const mockOnChange = vi.fn(); + const initialValue = [ + { + callback_name: "arize", + callback_type: "success", + callback_vars: { arize_space_id: "s", arize_otlp_protocol: "http/protobuf" }, + }, + ]; + + renderWithProviders(); + + await user.click(screen.getByRole("combobox", { name: "Event Type" })); + await user.click(await screen.findByRole("option", { name: "Success & Failure" })); + + const lastCall = mockOnChange.mock.calls[mockOnChange.mock.calls.length - 1]; + expect(lastCall[0][0].callback_type).toBe("success_and_failure"); + expect(lastCall[0][0].callback_vars).toEqual({ + arize_space_id: "s", + arize_otlp_protocol: "http/protobuf", + }); + }); + it("renders sampling rate inputs for the Arize callback and records changes", () => { const mockOnChange = vi.fn(); diff --git a/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx b/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx index 2f66e68eca4..73287c7abbf 100644 --- a/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx +++ b/ui/litellm-dashboard/src/components/team/LoggingSettings.tsx @@ -114,6 +114,21 @@ const LoggingSettings: React.FC = ({ [field]: callbackValue, callback_vars: {}, }; + } else if (field === "callback_type" && newValue === "failure") { + const callbackDisplayName = Object.entries(callback_map).find( + ([_, value]) => value === updatedConfigs[index].callback_name, + )?.[0]; + const successOnlyParams = new Set( + (callbackDisplayName && callbackInfo[callbackDisplayName]?.success_event_params) || [], + ); + const callbackVars = Object.fromEntries( + Object.entries(updatedConfigs[index].callback_vars).filter(([key]) => !successOnlyParams.has(key)), + ); + updatedConfigs[index] = { + ...updatedConfigs[index], + [field]: newValue, + callback_vars: callbackVars, + }; } else { updatedConfigs[index] = { ...updatedConfigs[index], @@ -196,6 +211,10 @@ const LoggingSettings: React.FC = ({ const dynamicParams = callbackInfo[callbackDisplayName]?.dynamic_params || {}; const paramOptions = callbackInfo[callbackDisplayName]?.dynamic_param_options || {}; + const successOnlyParams = + config.callback_type === "failure" + ? new Set(callbackInfo[callbackDisplayName]?.success_event_params ?? []) + : new Set(); if (Object.keys(dynamicParams).length === 0) return null; @@ -208,22 +227,24 @@ const LoggingSettings: React.FC = ({ Integration Parameters
- {Object.entries(dynamicParams).map(([paramName, paramType]) => ( -
- - {paramType === "number" && ( - Value must be between 0 and 1 - )} - {renderParamControl(config, configIndex, paramName, { - type: paramType, - options: paramType === "select" ? paramOptions[paramName] || [] : [], - })} -
- ))} + {Object.entries(dynamicParams) + .filter(([paramName]) => !successOnlyParams.has(paramName)) + .map(([paramName, paramType]) => ( +
+ + {paramType === "number" && ( + Value must be between 0 and 1 + )} + {renderParamControl(config, configIndex, paramName, { + type: paramType, + options: paramType === "select" ? paramOptions[paramName] || [] : [], + })} +
+ ))}
); From abc543e701707b7e95447cad5fe7a1b76ea96f1f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:50:25 -0700 Subject: [PATCH 2/5] feat(proxy): add opt-in vector_store_deny_by_default for least-privilege vector store access (#44244) * feat(proxy): add opt-in vector_store_deny_by_default for standalone virtual keys Adds general_settings.vector_store_deny_by_default (typed bool, default false). When enabled, a virtual key with no team must list the requested vector store in its object_permission.vector_stores; no permission record, null, or an empty list is denied with key_vector_store_access_denied. Omitted or false keeps the existing behavior, including nonempty allowlist enforcement. The master key is unchanged in both modes. Team keys and keyless callers are deferred to later increments of LIT-6035 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(proxy): require key and team vector store grants for team keys under vector_store_deny_by_default With the flag enabled, a virtual key on a team needs both its own grant and its team's grant for every requested vector store. A missing permission record, an empty list or an unresolved team grants nothing. Dashboard session keys and the master key keep their existing behavior, and flag-off behavior is unchanged Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(proxy): require user or team vector store grants for keyless requests under vector_store_deny_by_default Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): cover vector_store_deny_by_default through a real proxy with key, team and JWT identities Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(ui): regenerate schema.d.ts for vector_store_deny_by_default Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(auth): cover path and file_search vector store ids under vector_store_deny_by_default Strict mode now reads vector_store_ids and tools[].vector_store_ids from the request body without needing a vector store registry, so /v1/vector_stores/{id}/search and Responses file_search are checked. User grants load through the object permission cache, and the proxy admin user rebuild keeps object_permission_id Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): broadcast user entitlement cache eviction to every worker Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(auth): reuse VectorStoreRegistry id extraction for vector_store_deny_by_default Strict mode now uses get_vector_store_ids_to_run on an empty registry when none is loaded, instead of a parallel set of request-shape helpers, and vector_store_access_check documents the policy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): add strict vector store audit cells for routes, SDKs, workers and concurrency Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): consolidate vector_store_deny_by_default coverage to core cases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): reject invalid vector_store_deny_by_default at config load and return 400 for malformed vector_store_ids Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(auth): read vector store key and team grants through the object permission cache Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(auth): return typed permission rows in request flow vector store tests Co-authored-by: mrinal Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(auth): validate vector store ids and tools as immutable sequences Co-authored-by: mrinal Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mrinal Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/error_normalization.py | 1 + litellm/proxy/_types.py | 13 +- litellm/proxy/auth/auth_checks.py | 266 ++++++-- litellm/proxy/auth/user_api_key_auth.py | 3 + .../internal_user_endpoints.py | 6 +- .../management_endpoints/team_endpoints.py | 5 + litellm/proxy/proxy_server.py | 8 + .../test_object_permission_lookup.py | 10 +- .../test_rag_query_vector_store_allowlist.py | 349 +++++++++- ...st_auth_checks_object_access_and_lookup.py | 629 +++++++++++++++++- .../test_user_api_key_auth_request_flow.py | 219 ++++++ .../test_internal_user_endpoints.py | 28 + .../test_team_endpoints.py | 2 + .../proxy/proxy_server/test_proxy_config.py | 38 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 + 15 files changed, 1515 insertions(+), 68 deletions(-) diff --git a/litellm/litellm_core_utils/error_normalization.py b/litellm/litellm_core_utils/error_normalization.py index 726fdcd1540..4426d42f767 100644 --- a/litellm/litellm_core_utils/error_normalization.py +++ b/litellm/litellm_core_utils/error_normalization.py @@ -107,6 +107,7 @@ _PROXY_ERROR_TYPE_MAP: Final[Mapping[str, str]] = MappingProxyType( "key_vector_store_access_denied": PERMISSION_DENIED, "team_vector_store_access_denied": PERMISSION_DENIED, "org_vector_store_access_denied": PERMISSION_DENIED, + "user_vector_store_access_denied": PERMISSION_DENIED, "tool_access_denied": PERMISSION_DENIED, "team_member_permission_error": PERMISSION_DENIED, "not_found_error": RESOURCE_NOT_FOUND, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c1a97aa73c4..23cd64b8a49 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3022,6 +3022,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.", ) + vector_store_deny_by_default: bool = Field( + default=False, + description="When True, a vector store must be explicitly listed in object_permission.vector_stores: a virtual key needs its own grant plus its team's, a keyless team member needs the team's, and a user with neither needs their own. A missing permission record, an empty list, or an unresolved team grants nothing. Dashboard session keys are not yet covered", + ) missing_session_id: Literal["generate", "reject", "omit"] | None = Field( None, description="What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400; 'omit' leaves SpendLogs.session_id null, matching callbacks such as Langfuse that only record a client-established metadata.session_id. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.", @@ -4512,6 +4516,11 @@ class ProxyErrorTypes(str, enum.Enum): Organization does not have access to the vector store """ + user_vector_store_access_denied = "user_vector_store_access_denied" + """ + User does not have access to the vector store + """ + team_member_already_in_team = "team_member_already_in_team" """ Team member is already in team @@ -4546,7 +4555,7 @@ class ProxyErrorTypes(str, enum.Enum): @classmethod def get_vector_store_access_error_type_for_object( - cls, object_type: Literal["key", "team", "org"] + cls, object_type: Literal["key", "team", "org", "user"] ) -> "ProxyErrorTypes": """ Get the vector store access error type for object_type @@ -4557,6 +4566,8 @@ class ProxyErrorTypes(str, enum.Enum): return cls.team_vector_store_access_denied elif object_type == "org": return cls.org_vector_store_access_denied + elif object_type == "user": + return cls.user_vector_store_access_denied DB_CONNECTION_ERROR_TYPES: Final = ( diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 460b25ddac7..3ebd1a2cfe0 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -15,6 +15,7 @@ import re import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from functools import partial +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Generic, Literal, Optional, Protocol, TypeAlias @@ -34,6 +35,7 @@ from litellm.constants import ( DEFAULT_MAX_RECURSE_DEPTH, EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE, END_USER_RESTRICTED_REGISTRY_MAX_SIZE, + LITELLM_PROXY_MASTER_KEY_ALIAS, MODEL_ACCESS_GROUP_REGISTRY_MAX_SIZE, REGISTRY_ERROR_NEGATIVE_CACHE_TTL, TAG_REGISTRY_MAX_SIZE, @@ -44,7 +46,9 @@ from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.models.project import LiteLLM_ProjectTable from litellm.proxy._types import ( RBAC_ROLES, + UI_TEAM_ID, CallInfo, + ConfigGeneralSettings, LiteLLM_AccessGroupTable, LiteLLM_BudgetTable, LiteLLM_EndUserTable, @@ -313,12 +317,6 @@ class _VectorStorePermissionsRow(Protocol): def vector_stores(self) -> Sequence[str] | None: ... -def _object_permission_table( - repo: _PrismaTableHolder[_VectorStorePermissionsRow], -) -> _PrismaAuthTable[_VectorStorePermissionsRow]: - return _DeadlineBoundedTable(repo.table, "object_permission") - - class _PrismaTagRow(Protocol): tag_name: str @@ -1353,6 +1351,8 @@ async def common_checks( request_body=request_body, team_object=team_object, valid_token=valid_token, + user_object=user_object, + deny_by_default=_vector_store_deny_by_default(_typed_request_body(general_settings)), ) # 12. [OPTIONAL] Tool allowlist - key/team allowed_tools (no DB in hot path) @@ -6792,17 +6792,210 @@ def _get_rag_query_vector_store_id(request_body: Mapping[str, object]) -> str | return vector_store_id if isinstance(vector_store_id, str) and vector_store_id else None +GrantLayer = Literal["key", "team", "user"] + + +def _is_strict_grant_identity(valid_token: UserAPIKeyAuth | None) -> bool: + return ( + valid_token is not None + and valid_token.api_key != LITELLM_PROXY_MASTER_KEY_ALIAS + and valid_token.team_id != UI_TEAM_ID + ) + + +def _strict_grant_layers( + deny_by_default: bool, valid_token: UserAPIKeyAuth | None, team_object: LiteLLM_TeamTable | None +) -> frozenset[GrantLayer]: + """ + The identities that must each grant an object under a deny-by-default policy: a virtual key and its team, + a keyless team member's team, or a keyless user's own grant. The master key and dashboard sessions need none + """ + if not deny_by_default or valid_token is None or not _is_strict_grant_identity(valid_token): + return frozenset() + virtual_key: Final = valid_token.via_virtual_key and not valid_token.is_session_token + has_team: Final = team_object is not None or valid_token.team_id is not None + if not virtual_key and not has_team: + return frozenset(("user",)) + return frozenset(layer for layer, required in (("key", virtual_key), ("team", has_team)) if required) + + +async def _identity_grants( + valid_token: UserAPIKeyAuth | None, + team_object: LiteLLM_TeamTable | None, + user_object: LiteLLM_UserTable | None, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, +) -> tuple[tuple[GrantLayer, LiteLLM_ObjectPermissionTable | None], ...]: + return ( + ( + "key", + None + if valid_token is None + else await _cached_object_permission( + valid_token.object_permission_id, valid_token.object_permission, prisma_client, user_api_key_cache + ), + ), + ( + "team", + None + if team_object is None + else await _cached_object_permission( + team_object.object_permission_id, team_object.object_permission, prisma_client, user_api_key_cache + ), + ), + ( + "user", + None + if user_object is None + else await _cached_object_permission( + user_object.object_permission_id, user_object.object_permission, prisma_client, user_api_key_cache + ), + ), + ) + + +def _vector_store_deny_by_default(general_settings: Mapping[str, object]) -> bool: + """ + Startup rejects a non-boolean value from the config file. A non-boolean value that reaches + general_settings another way enables the policy, so only vector store requests are denied. + """ + try: + return ConfigGeneralSettings.model_validate( + MappingProxyType( + {"vector_store_deny_by_default": general_settings.get("vector_store_deny_by_default", False)} + ) + ).vector_store_deny_by_default + except ValidationError: + return True + + +_VECTOR_STORE_IDS_ADAPTER: Final[TypeAdapter[Sequence[str]]] = TypeAdapter(Sequence[str]) +_TOOLS_ADAPTER: Final[TypeAdapter[Sequence[object]]] = TypeAdapter(Sequence[object]) +_TOOL_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def _validated_vector_store_ids(value: object) -> tuple[str, ...]: + if value is None: + return () + try: + return tuple(_VECTOR_STORE_IDS_ADAPTER.validate_python(value, strict=True)) + except ValidationError: + raise _malformed_vector_store_ids() from None + + +def _malformed_vector_store_ids() -> ProxyException: + return ProxyException( + message="vector_store_ids must be a list of strings", + type="invalid_request_error", + param="vector_store_ids", + code=status.HTTP_400_BAD_REQUEST, + ) + + +def _tools(tools: object) -> tuple[object, ...]: + try: + return tuple(_TOOLS_ADAPTER.validate_python(tools, strict=True)) + except ValidationError: + return () + + +def _tool_vector_store_ids(tool: object) -> tuple[str, ...]: + try: + tool_fields: Final = _TOOL_ADAPTER.validate_python(tool, strict=True) + except ValidationError: + return () + return _validated_vector_store_ids(tool_fields.get("vector_store_ids")) + + +def _strict_requested_vector_store_ids(request_body: Mapping[str, object]) -> tuple[str, ...]: + """ + Same fields VectorStoreRegistry.get_vector_store_ids_to_run reads, but a vector_store_ids that is + not a list of strings is a 400 instead of being skipped or iterated, and tools that are not + objects name no store. + """ + return tuple( + chain( + _validated_vector_store_ids(request_body.get("vector_store_ids")), + chain.from_iterable(_tool_vector_store_ids(tool) for tool in _tools(request_body.get("tools"))), + ) + ) + + +def _require_vector_store_grant( + object_type: GrantLayer, + vector_store_ids_to_run: Sequence[str], + object_permission: _VectorStorePermissionsRow | None, +) -> None: + if object_permission is None or not object_permission.vector_stores: + raise ProxyException( + message=f"{object_type.capitalize()} not allowed to access vector store. Tried to access {vector_store_ids_to_run[0]}. vector_store_deny_by_default is enabled and the {object_type} has no vector store grants", + type=ProxyErrorTypes.get_vector_store_access_error_type_for_object(object_type), + param="vector_store", + code=status.HTTP_401_UNAUTHORIZED, + ) + _can_object_call_vector_stores( + object_type=object_type, + vector_store_ids_to_run=vector_store_ids_to_run, + object_permissions=object_permission, + ) + + +async def _cached_object_permission( + object_permission_id: str | None, + loaded: LiteLLM_ObjectPermissionTable | None, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, +) -> LiteLLM_ObjectPermissionTable | None: + """ + The grant row auth already attached to the key, team or user, else the cached row by id. Both are + evicted on every worker when the grant changes, so the request path never reads the table directly. + """ + if object_permission_id is None: + return None + if loaded is not None: + return loaded + return await get_object_permission( + object_permission_id=object_permission_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) + + async def vector_store_access_check( request_body: dict, team_object: LiteLLM_TeamTable | None, valid_token: UserAPIKeyAuth | None, + *, + user_object: LiteLLM_UserTable | None = None, + deny_by_default: bool = False, ): """ - Checks if the object (key, team, org) has access to the vector store. + Checks whether the caller may use every vector store the request names. - Raises ProxyException if the object (key, team, org) cannot access the specific vector store. + Requested stores come from `vector_store_ids`, `tools[].vector_store_ids` and the RAG + `retrieval_config.vector_store_id`. Grants come from each identity's + `object_permission.vector_stores`. + + With `deny_by_default=False` (legacy), a key or team only restricts access when its list is + nonempty, and stores are only read from the request when a vector store registry is loaded. + + With `deny_by_default=True` (`general_settings.vector_store_deny_by_default`), stores are read + even without a registry, and a missing record, `null` or `[]` grants nothing: + + - virtual key: the key must grant every store, and so must its team when it has one, even if + the team failed to load + - keyless team member (JWT, `lite login` session token): only the resolved team is checked + - keyless user with no team: the user's own grant is checked + - master key and dashboard sessions: legacy behavior + + The user's personal grant is only consulted in the keyless no-team case, so it can neither + rescue nor restrict a key or team request. + + Raises ProxyException (401, `{key,team,user}_vector_store_access_denied`) on the first identity + that does not grant a requested store, and with the flag on, ProxyException (400, + `invalid_request_error`) when a `vector_store_ids` field is not a list of strings. """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache ######################################################### # Get the vector store the user is trying to access @@ -6812,12 +7005,17 @@ async def vector_store_access_check( return True registry_ids: Final = ( - litellm.vector_store_registry.get_vector_store_ids_to_run( - non_default_params=request_body, tools=request_body.get("tools", None) + _strict_requested_vector_store_ids(_typed_request_body(request_body)) + if deny_by_default + else ( + litellm.vector_store_registry.get_vector_store_ids_to_run( + non_default_params=request_body, tools=request_body.get("tools", None) + ) + if litellm.vector_store_registry is not None + else None ) - if litellm.vector_store_registry is not None - else None - ) or () + or () + ) rag_vector_store_id: Final = _get_rag_query_vector_store_id(_typed_request_body(request_body)) rag_ids: Final = (rag_vector_store_id,) if rag_vector_store_id is not None else () vector_store_ids_to_run: Final = tuple(dict.fromkeys((*registry_ids, *rag_ids))) @@ -6828,38 +7026,28 @@ async def vector_store_access_check( ######################################################### # Check if the object (key, team, org) has access to the vector store ######################################################### - # Check if the key can access the vector store - if valid_token is not None and valid_token.object_permission_id is not None: - key_object_permission: Final = await _object_permission_table( - ObjectPermissionRepository(prisma_client) - ).find_unique( - where={"object_permission_id": valid_token.object_permission_id}, - ) - if key_object_permission is not None: + strict_layers: Final = _strict_grant_layers(deny_by_default, valid_token, team_object) + grants: Final = await _identity_grants( + valid_token, + team_object, + user_object if "user" in strict_layers else None, + prisma_client, + user_api_key_cache, + ) + for layer, grant in grants: + if layer in strict_layers: + _require_vector_store_grant(layer, vector_store_ids_to_run, grant) + elif grant is not None: _can_object_call_vector_stores( - object_type="key", + object_type=layer, vector_store_ids_to_run=vector_store_ids_to_run, - object_permissions=key_object_permission, - ) - - # Check if the team can access the vector store - if team_object is not None and team_object.object_permission_id is not None: - team_object_permission: Final = await _object_permission_table( - ObjectPermissionRepository(prisma_client) - ).find_unique( - where={"object_permission_id": team_object.object_permission_id}, - ) - if team_object_permission is not None: - _can_object_call_vector_stores( - object_type="team", - vector_store_ids_to_run=vector_store_ids_to_run, - object_permissions=team_object_permission, + object_permissions=grant, ) return True def _can_object_call_vector_stores( - object_type: Literal["key", "team", "org"], + object_type: Literal["key", "team", "org", "user"], vector_store_ids_to_run: Sequence[str], object_permissions: _VectorStorePermissionsRow | None, ): diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index adb6b86321d..65f9b09a4d3 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -3063,6 +3063,9 @@ async def _run_centralized_common_checks( user_id=user_api_key_auth_obj.user_id or litellm_proxy_admin_name, user_role=LitellmUserRoles.PROXY_ADMIN, spend=user_object.spend if user_object is not None else 0.0, + object_permission_id=( + user_object.object_permission_id if isinstance(user_object, LiteLLM_UserTable) else None + ), ) if project_object is not None: diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 69efb9c209f..98df4ae46b1 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -1458,11 +1458,7 @@ async def _invalidate_cached_user_entitlement(user_id: str | None, object_permis *(object_permission_cache_key(permission_id) for permission_id in dict.fromkeys(object_permission_ids)), *((user_object_permission_id_cache_key(user_id), user_id) if user_id is not None else ()), ) - for key in keys: - try: - await user_api_key_cache.async_delete_cache(key=key) - except Exception as e: # noqa: BLE001 # a cache we cannot clear still expires; never fail the write - verbose_proxy_logger.warning("Failed to invalidate cached entitlement key %r: %s", key, e) + await evict_and_broadcast(cache_keys=keys, user_api_key_cache=user_api_key_cache) async def _update_single_user_helper( diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 818c0590707..aeb317c8905 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -165,6 +165,7 @@ from litellm.proxy.management_helpers.object_permission_utils import ( _set_object_permission, enforce_all_proxy_mcp_servers_grant_is_admin_only, handle_update_object_permission_common, + invalidate_cached_object_permissions, ) from litellm.proxy.management_helpers.team_member_permission_checks import ( TeamMemberPermissionChecks, @@ -2546,6 +2547,10 @@ async def update_team( verbose_proxy_logger.info("Successfully updated team - %s, info", team_row.team_id) await sync_team_access_group_membership(prisma_client=prisma_client, team_id=team_row.team_id) + await invalidate_cached_object_permissions( + object_permission_ids=(existing_team.object_permission_id, team_row.object_permission_id), + user_api_key_cache=user_api_key_cache, + ) await _refresh_cached_team( team_row=team_row, user_api_key_cache=user_api_key_cache, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d1147c48558..6d522759eb7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6713,6 +6713,14 @@ class ProxyConfig: if general_settings is None: general_settings = {} + typed_general_settings: Final = _GENERAL_SETTINGS_VIEW.validate_python(general_settings) + if "vector_store_deny_by_default" in typed_general_settings: + ConfigGeneralSettings.model_validate( + MappingProxyType( + {"vector_store_deny_by_default": typed_general_settings["vector_store_deny_by_default"]} + ) + ) + if general_settings.get("mcp_advertised_versions") is not None: from litellm.types.mcp import MCPAdvertisedVersions diff --git a/tests/integration/authorization/test_object_permission_lookup.py b/tests/integration/authorization/test_object_permission_lookup.py index 21fba30c165..a977243e642 100644 --- a/tests/integration/authorization/test_object_permission_lookup.py +++ b/tests/integration/authorization/test_object_permission_lookup.py @@ -57,18 +57,18 @@ def _assert_forbidden_vector_store_denied(gateway: Gateway, model: str, key: str @pytest.mark.covers("authorization.vector_store.plain_request_skips_object_permission_lookup") -def test_chat_request_without_vector_stores_does_not_read_object_permission_table(gateway: Gateway) -> None: +def test_requests_do_not_read_object_permission_table_once_the_grant_is_cached(gateway: Gateway) -> None: with gateway.scenario() as scenario: model: Final = scenario.model() key: Final = scenario.key(models=[model], object_permission={"vector_stores": ["vs_allowed"]}) - _assert_plain_chat_served(gateway, model, key) - before_control: Final = _object_permission_reads() + before_first_use: Final = _object_permission_reads() _assert_forbidden_vector_store_denied(gateway, model, key) - eventually(_object_permission_reads, lambda reads: reads > before_control, seconds=15) + eventually(_object_permission_reads, lambda reads: reads > before_first_use, seconds=15) baseline: Final = _settled_object_permission_reads(_object_permission_reads(), time.monotonic()) for _ in range(PLAIN_REQUESTS): _assert_plain_chat_served(gateway, model, key) + _assert_forbidden_vector_store_denied(gateway, model, key) after: Final = _settled_object_permission_reads(_object_permission_reads(), time.monotonic()) assert after - baseline < PLAIN_REQUESTS, ( - f"{PLAIN_REQUESTS} plain chat requests added {after - baseline} object permission reads" + f"{2 * PLAIN_REQUESTS} requests after the first added {after - baseline} object permission reads" ) diff --git a/tests/integration/authorization/test_rag_query_vector_store_allowlist.py b/tests/integration/authorization/test_rag_query_vector_store_allowlist.py index 896c88c68bb..d3f16e46a1e 100644 --- a/tests/integration/authorization/test_rag_query_vector_store_allowlist.py +++ b/tests/integration/authorization/test_rag_query_vector_store_allowlist.py @@ -1,5 +1,8 @@ from __future__ import annotations +import json +import os +import time import uuid from collections.abc import Iterator, Mapping from pathlib import Path @@ -7,16 +10,22 @@ from types import MappingProxyType from typing import Final, Literal, TypeAlias import httpx +import jwt import pytest import yaml -from integration._support.client import Gateway, Scenario, gateway_from_environment, object_value +from cryptography.hazmat.primitives.asymmetric import rsa +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, object_value from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server from integration.authorization._guardrail_opt_out import upstream_observations from pydantic import JsonValue +from redis import Redis CONFIG_STORE_ID: Final = "vs_integration_config_store" PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" REMOVE_OPENAI_API_BASE: Final = ("OPENAI_API_BASE",) +JWT_KEY_ID: Final = "integration-vector-store-jwt-key" +AUTH_CACHE_INVALIDATION_CHANNEL: Final = "litellm_proxy.auth_cache_invalidation" JsonObject: TypeAlias = dict[str, JsonValue] @@ -99,6 +108,211 @@ def no_registry_gateways(tmp_path_factory: pytest.TempPathFactory) -> Iterator[t yield no_registry_gateway, upstream_gateway +class StrictGateway: + def __init__( + self, + gateway: Gateway, + upstream: Gateway, + signing_key: rsa.RSAPrivateKey, + config: Path, + environment: Mapping[str, str], + ) -> None: + self.gateway: Final = gateway + self.upstream: Final = upstream + self._signing_key: Final = signing_key + self.config: Final = config + self.environment: Final = environment + + def jwt(self, subject: str, groups: tuple[str, ...] = ()) -> str: + claims: Final[JsonObject] = { + "sub": subject, + "groups": _json_array(*groups), + "iat": int(time.time()), + "exp": int(time.time()) + 300, + } + return jwt.encode(claims, self._signing_key, algorithm="RS256", headers={"kid": JWT_KEY_ID}) + + +def _strict_config(directory: Path, *, deny_by_default: bool = True) -> Path: + config: Final = object_value(yaml.safe_load(PROXY_CONFIG.read_text())) + general_settings: Final = object_value(config["general_settings"]) + strict: Final[JsonObject] = { + **config, + "general_settings": { + **general_settings, + "vector_store_deny_by_default": deny_by_default, + "enable_jwt_auth": True, + "litellm_jwtauth": { + "user_id_jwt_field": "sub", + "team_ids_jwt_field": "groups", + "team_allowed_routes": _json_array("openai_routes", "info_routes", "/v1/rag/query"), + }, + }, + } + path: Final = directory / f"proxy_vector_store_deny_by_default_{deny_by_default}.yaml" + path.write_text(yaml.safe_dump(strict)) + return path + + +@pytest.fixture(scope="module") +def strict_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[StrictGateway]: + signing_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + public_jwk: Final = jwt.algorithms.RSAAlgorithm.to_jwk(signing_key.public_key()) + jwks_body: Final = json.dumps({"keys": [{**json.loads(public_jwk), "kid": JWT_KEY_ID}]}).encode() + + def respond(request: Request) -> Reply: + assert request.method == "GET", request + return Reply(body=jwks_body) + + with gateway_from_environment() as upstream_gateway, wire_server(respond) as jwks: + directory: Final = tmp_path_factory.mktemp("rag_query_deny_by_default") + config: Final = _strict_config(directory) + environment: Final = MappingProxyType({**_openai_environment(upstream_gateway), "JWT_PUBLIC_KEY_URL": jwks.url}) + with owned_proxy( + upstream_gateway, + directory, + environment, + config=config, + remove_environment=REMOVE_OPENAI_API_BASE, + ) as gateway: + yield StrictGateway(gateway, upstream_gateway, signing_key, config, environment) + + +StrictCase: TypeAlias = Literal[ + "standalone_key_no_permission", + "team_key_empty_key_grants", + "team_key_empty_team_grants", + "multi_store_one_ungranted", + "jwt_user_without_grant", +] + + +def _strict_denied_request( + strict: StrictGateway, scenario: Scenario, case: StrictCase, model: str, marker: str, store_id: str +) -> tuple[httpx.Response, str]: + models: Final = _json_array(model) + granted: Final = _permission_for_stores(store_id) + empty: Final = _permission_for_stores() + if case == "standalone_key_no_permission": + standalone_key: Final = scenario.key(models=models) + return strict.gateway.request( + "POST", "/v1/rag/query", _rag_query_body(model, marker, store_id), key=standalone_key + ), ("key_vector_store_access_denied") + if case in ("team_key_empty_key_grants", "team_key_empty_team_grants"): + key_grants_store: Final = case == "team_key_empty_team_grants" + team: Final = scenario.team(models=models, object_permission=empty if key_grants_store else granted) + team_key: Final = scenario.key( + team_id=team, models=models, object_permission=granted if key_grants_store else empty + ) + error_type: Final = "team_vector_store_access_denied" if key_grants_store else "key_vector_store_access_denied" + return strict.gateway.request( + "POST", "/v1/rag/query", _rag_query_body(model, marker, store_id), key=team_key + ), (error_type) + if case == "multi_store_one_ungranted": + partial_key: Final = scenario.key(models=models, object_permission=_permission_for_stores(store_id)) + body: Final[JsonObject] = { + **_rag_query_body(model, marker, store_id), + "tools": _json_array({"type": "file_search", "vector_store_ids": _json_array(CONFIG_STORE_ID)}), + } + return strict.gateway.request( + "POST", "/v1/chat/completions", body, key=partial_key + ), "key_vector_store_access_denied" + user: Final = scenario.user(user_role="internal_user") + return ( + strict.gateway.request("POST", "/v1/rag/query", _rag_query_body(model, marker, store_id), key=strict.jwt(user)), + "user_vector_store_access_denied", + ) + + +@pytest.mark.parametrize( + "case", + ( + "standalone_key_no_permission", + "team_key_empty_key_grants", + "team_key_empty_team_grants", + "multi_store_one_ungranted", + "jwt_user_without_grant", + ), +) +def test_deny_by_default_rejects_ungranted_store_before_upstream_search( + strict_gateway: StrictGateway, case: StrictCase +) -> None: + with strict_gateway.gateway.scenario() as scenario: + model: Final = scenario.model() + store_id: Final = f"vs_unregistered_{uuid.uuid4().hex}" + marker: Final = f"lit6035 deny by default {case} {uuid.uuid4().hex}" + + response, error_type = _strict_denied_request(strict_gateway, scenario, case, model, marker, store_id) + observations: Final = tuple( + observation + for observation in upstream_observations(strict_gateway.upstream) + if marker in str(observation["body"]) + ) + + assert response.status_code == 401, f"{response.text}; scripted_upstream_observations={observations!r}" + assert response.json()["error"]["type"] == error_type, response.text + assert observations == () + + +GrantedCase: TypeAlias = Literal[ + "standalone_key_granted_registered_store", + "team_key_both_grant_unregistered_store", + "jwt_team_member_team_grant_only", + "jwt_user_personal_grant", + "master_key_without_grants", +] + + +def _strict_granted_request( + strict: StrictGateway, scenario: Scenario, case: GrantedCase, model: str, marker: str, store_id: str +) -> httpx.Response: + models: Final = _json_array(model) + granted: Final = _permission_for_stores(store_id) + body: Final = _rag_query_body(model, marker, store_id) + if case in ("standalone_key_granted_registered_store", "team_key_both_grant_unregistered_store"): + key_team: Final = scenario.team(models=models, object_permission=granted) if case.startswith("team") else None + granted_key: Final = scenario.key( + models=models, object_permission=granted, **({} if key_team is None else {"team_id": key_team}) + ) + return strict.gateway.request("POST", "/v1/rag/query", body, key=granted_key) + if case == "jwt_team_member_team_grant_only": + member_team: Final = scenario.team(models=models, object_permission=granted) + member: Final = scenario.member(member_team) + return strict.gateway.request("POST", "/v1/rag/query", body, key=strict.jwt(member, (member_team,))) + if case == "jwt_user_personal_grant": + user: Final = scenario.user(user_role="internal_user", object_permission=granted) + return strict.gateway.request("POST", "/v1/rag/query", body, key=strict.jwt(user)) + return strict.gateway.request("POST", "/v1/rag/query", body) + + +@pytest.mark.parametrize( + "case", + ( + "standalone_key_granted_registered_store", + "team_key_both_grant_unregistered_store", + "jwt_team_member_team_grant_only", + "jwt_user_personal_grant", + "master_key_without_grants", + ), +) +def test_deny_by_default_searches_explicitly_granted_store(strict_gateway: StrictGateway, case: GrantedCase) -> None: + with strict_gateway.gateway.scenario() as scenario: + model: Final = scenario.model() + store_id: Final = ( + CONFIG_STORE_ID + if case == "standalone_key_granted_registered_store" + else f"vs_unregistered_{uuid.uuid4().hex}" + ) + marker: Final = f"lit6035 granted {case} {uuid.uuid4().hex}" + + response: Final = _strict_granted_request(strict_gateway, scenario, case, model, marker, store_id) + assert response.status_code == 200, response.text + + searches: Final = _searches_for_marker(strict_gateway.upstream, marker, store_id) + assert len(searches) == 1, searches + assert marker in str(object_value(searches[0]["body"])["query"]), searches + + @pytest.mark.parametrize( ("scope", "error_type"), (("key", "key_vector_store_access_denied"), ("team", "team_vector_store_access_denied")), @@ -220,3 +434,136 @@ def test_rag_query_alias_denies_store_when_team_allowlist_excludes(gateway: Gate assert response.status_code == 401, response.text assert response.json()["error"]["type"] == "team_vector_store_access_denied", response.text assert _searches_for_marker(gateway, marker) == () + + +def test_explicit_false_flag_keeps_legacy_vector_store_outcomes(tmp_path: Path) -> None: + with gateway_from_environment() as upstream_gateway: + with owned_proxy( + upstream_gateway, + tmp_path, + _openai_environment(upstream_gateway), + config=_strict_config(tmp_path, deny_by_default=False), + remove_environment=REMOVE_OPENAI_API_BASE, + ) as gateway: + with gateway.scenario() as scenario: + model: Final = scenario.model() + allowed_marker: Final = f"flag-false-allowed-{uuid.uuid4().hex}" + denied_marker: Final = f"flag-false-denied-{uuid.uuid4().hex}" + no_permission_key: Final = scenario.key(models=_json_array(model)) + excluding_key: Final = scenario.key( + models=_json_array(model), object_permission=_permission_for_stores("vs_some_other_store") + ) + + allowed: Final = _rag_query(gateway, model, allowed_marker, no_permission_key) + denied: Final = _rag_query(gateway, model, denied_marker, excluding_key) + + assert allowed.status_code == 200, allowed.text + assert len(_searches_for_marker(upstream_gateway, allowed_marker)) == 1 + assert denied.status_code == 401, denied.text + assert denied.json()["error"]["type"] == "key_vector_store_access_denied" + assert _searches_for_marker(upstream_gateway, denied_marker) == () + + +def _strict_no_registry_config(directory: Path) -> Path: + config: Final = object_value(yaml.safe_load(_no_registry_config(directory).read_text())) + general_settings: Final = object_value(config["general_settings"]) + strict: Final[JsonObject] = { + **config, + "general_settings": {**general_settings, "vector_store_deny_by_default": True}, + } + path: Final = directory / "proxy_vector_store_deny_by_default_no_registry.yaml" + path.write_text(yaml.safe_dump(strict)) + return path + + +def test_deny_by_default_without_registry_checks_search_route_and_file_search_tools(tmp_path: Path) -> None: + with gateway_from_environment() as upstream_gateway: + with owned_proxy( + upstream_gateway, + tmp_path, + _openai_environment(upstream_gateway), + config=_strict_no_registry_config(tmp_path), + remove_environment=REMOVE_OPENAI_API_BASE, + ) as gateway: + with gateway.scenario() as scenario: + model: Final = scenario.model() + store_id: Final = f"vs_unregistered_{uuid.uuid4().hex}" + search_marker: Final = f"lit6035 no registry search {uuid.uuid4().hex}" + responses_marker: Final = f"lit6035 no registry responses {uuid.uuid4().hex}" + granted_marker: Final = f"lit6035 no registry granted search {uuid.uuid4().hex}" + ungranted_key: Final = scenario.key(models=_json_array(model)) + granted_key: Final = scenario.key( + models=_json_array(model), object_permission=_permission_for_stores(store_id) + ) + + search_denied: Final = gateway.request( + "POST", f"/v1/vector_stores/{store_id}/search", {"query": search_marker}, key=ungranted_key + ) + responses_denied: Final = gateway.request( + "POST", + "/v1/responses", + { + "model": model, + "input": responses_marker, + "tools": _json_array({"type": "file_search", "vector_store_ids": _json_array(store_id)}), + }, + key=ungranted_key, + ) + search_granted: Final = gateway.request( + "POST", f"/v1/vector_stores/{store_id}/search", {"query": granted_marker}, key=granted_key + ) + + observations: Final = upstream_observations(upstream_gateway) + denied_observations: Final = tuple( + observation + for observation in observations + if search_marker in str(observation["body"]) or responses_marker in str(observation["body"]) + ) + granted_searches: Final = tuple( + observation + for observation in observations + if observation["path"] == f"/vector_stores/{store_id}/search" + and granted_marker in str(observation["body"]) + ) + assert search_denied.status_code == 401, search_denied.text + assert search_denied.json()["error"]["type"] == "key_vector_store_access_denied", search_denied.text + assert responses_denied.status_code == 401, responses_denied.text + assert responses_denied.json()["error"]["type"] == "key_vector_store_access_denied", responses_denied.text + assert denied_observations == () + assert search_granted.status_code == 200, search_granted.text + assert len(granted_searches) == 1, granted_searches + + +def _auth_cache_subscribers() -> int: + with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache: + return int(cache.pubsub_numsub(AUTH_CACHE_INVALIDATION_CHANNEL)[0][1]) + + +def test_revoked_user_grant_stops_working_on_another_proxy(strict_gateway: StrictGateway, tmp_path: Path) -> None: + subscribers_before_peer: Final = _auth_cache_subscribers() + with ( + owned_proxy( + strict_gateway.upstream, + tmp_path, + strict_gateway.environment, + config=strict_gateway.config, + remove_environment=REMOVE_OPENAI_API_BASE, + ) as peer, + strict_gateway.gateway.scenario() as scenario, + ): + eventually(_auth_cache_subscribers, lambda count: count > subscribers_before_peer) + model: Final = scenario.model() + store_id: Final = f"vs_unregistered_{uuid.uuid4().hex}" + user: Final = scenario.user(user_role="internal_user", object_permission=_permission_for_stores(store_id)) + token: Final = strict_gateway.jwt(user) + + def peer_status() -> int: + marker: Final = f"lit6035 revoked user grant {uuid.uuid4().hex}" + return peer.request( + "POST", "/v1/rag/query", _rag_query_body(model, marker, store_id), key=token + ).status_code + + assert eventually(peer_status, lambda status: status == 200, seconds=30, return_last_on_timeout=True) == 200 + strict_gateway.gateway.post("/user/update", {"user_id": user, "object_permission": _permission_for_stores()}) + + assert eventually(peer_status, lambda status: status == 401, seconds=10, return_last_on_timeout=True) == 401 diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index 9ce23c442f6..e855d8cd346 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -4,7 +4,7 @@ import json import re import sys import time -from collections.abc import Iterator, Mapping +from collections.abc import Awaitable, Iterator, Mapping from types import SimpleNamespace from typing import TYPE_CHECKING, Final, Literal, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -61,6 +61,7 @@ from litellm.proxy.auth.auth_checks import ( _check_agent_caller_model_access, _virtual_key_max_budget_check, _virtual_key_soft_budget_check, + common_checks, get_key_object, get_user_object, invalidate_team_member_spend_state, @@ -74,6 +75,7 @@ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.constants import ( DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, END_USER_RESTRICTED_REGISTRY_MAX_SIZE, + LITELLM_PROXY_MASTER_KEY_ALIAS, PROXY_DB_LOOKUP_MAX_CONCURRENCY, REGISTRY_ERROR_NEGATIVE_CACHE_TTL, TAG_REGISTRY_MAX_SIZE, @@ -90,6 +92,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, end_user_cache_key, end_user_restricted_registry_cache_key, + object_permission_cache_key, tag_cache_key, tag_registry_cache_key, ) @@ -1646,8 +1649,9 @@ async def test_vector_store_access_check_with_permissions(): ) mock_prisma_client = MagicMock() - mock_permissions = MagicMock() - mock_permissions.vector_stores = ["store-1", "store-2"] + mock_permissions = LiteLLM_ObjectPermissionTable( + object_permission_id="perm-123", vector_stores=["store-1", "store-2"] + ) mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=mock_permissions) mock_vector_store_registry = MagicMock() @@ -1688,12 +1692,12 @@ async def test_vector_store_access_check_with_team_permissions(): request_body = {} valid_token = UserAPIKeyAuth(token="team-test-token", object_permission_id=None) - team_object = MagicMock() - team_object.object_permission_id = "team-permission" + team_object = LiteLLM_TeamTable(team_id="team-1", object_permission_id="team-permission") mock_prisma_client = MagicMock() - team_permissions = MagicMock() - team_permissions.vector_stores = ["team-store-allowed"] + team_permissions = LiteLLM_ObjectPermissionTable( + object_permission_id="team-permission", vector_stores=["team-store-allowed"] + ) mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=team_permissions) mock_vector_store_registry = MagicMock() @@ -1755,12 +1759,12 @@ async def test_vector_store_access_check_enforces_team_allowlist_for_rag_query( } valid_token = UserAPIKeyAuth(token="team-test-token", object_permission_id=None) - team_object = MagicMock() - team_object.object_permission_id = "team-permission" + team_object = LiteLLM_TeamTable(team_id="team-1", object_permission_id="team-permission") mock_prisma_client = MagicMock() - team_permissions = MagicMock() - team_permissions.vector_stores = ["KBALLOWED123"] + team_permissions = LiteLLM_ObjectPermissionTable( + object_permission_id="team-permission", vector_stores=["KBALLOWED123"] + ) mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=team_permissions) with ( @@ -1786,6 +1790,592 @@ async def test_vector_store_access_check_enforces_team_allowlist_for_rag_query( assert exc_info.value.type == expected_error_type +_KEY_DENIED: Final = ProxyErrorTypes.key_vector_store_access_denied +_TEAM_DENIED: Final = ProxyErrorTypes.team_vector_store_access_denied +_USER_DENIED: Final = ProxyErrorTypes.user_vector_store_access_denied + + +def _virtual_key( + object_permission_id: str | None = None, + api_key: str = "sk-standalone", + user_role: LitellmUserRoles | None = None, + team_id: str | None = None, +) -> UserAPIKeyAuth: + key: Final = UserAPIKeyAuth( + api_key=api_key, + user_id="key-owner", + user_role=user_role, + team_id=team_id, + object_permission_id=object_permission_id, + ) + key.via_virtual_key = True + return key + + +def _permission_row( + object_permission_id: str, permission: SimpleNamespace | LiteLLM_ObjectPermissionTable | None +) -> LiteLLM_ObjectPermissionTable | None: + if isinstance(permission, SimpleNamespace): + return LiteLLM_ObjectPermissionTable( + object_permission_id=object_permission_id, vector_stores=permission.vector_stores + ) + return permission + + +async def _common_checks_for_rag_query( + general_settings: Mapping[str, object], + valid_token: UserAPIKeyAuth, + key_permission: SimpleNamespace | None, + team_object: LiteLLM_TeamTable | None = None, + team_permission: SimpleNamespace | None = None, + user_object: LiteLLM_UserTable | None = None, + user_permission: SimpleNamespace | LiteLLM_ObjectPermissionTable | None = None, + request_body: Mapping[str, object] | None = None, + user_api_key_cache: UserApiKeyCache | None = None, + vector_store_registry: VectorStoreRegistry | None = None, +) -> bool: + permissions: Final = { + "key-permission": _permission_row("key-permission", key_permission), + "team-permission": _permission_row("team-permission", team_permission), + "user-permission": _permission_row("user-permission", user_permission), + } + mock_prisma_client: Final = MagicMock() + mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock( + side_effect=lambda where: permissions.get(where["object_permission_id"]) + ) + mock_prisma_client.db.litellm_teammembership.find_unique = AsyncMock(return_value=None) + body: Final = ( + dict(request_body) + if request_body is not None + else { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "what is in this KB?"}], + "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"}, + } + ) + with ( + patch( # test-quality-ok: production auth reads these module globals; no dependency injection seam exists + "litellm.proxy.proxy_server.prisma_client", mock_prisma_client + ), + patch( # test-quality-ok: production auth reads this module global; no dependency injection seam exists + "litellm.vector_store_registry", vector_store_registry + ), + patch( # test-quality-ok: production auth reads this module global; no dependency injection seam exists + "litellm.proxy.proxy_server.user_api_key_cache", + UserApiKeyCache() if user_api_key_cache is None else user_api_key_cache, + ), + ): + return await common_checks( + request_body=body, + team_object=team_object, + user_object=user_object, + end_user_object=None, + global_proxy_spend=None, + general_settings=dict(general_settings), + route="/v1/rag/query", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=valid_token, + request=MagicMock(spec=Request), + ) + + +async def _assert_rag_query_outcome(request: Awaitable[bool], denied_by: ProxyErrorTypes | None) -> None: + if denied_by is None: + assert await request is True + return + with pytest.raises(ProxyException) as exc_info: + await request + assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == (denied_by, "vector_store", "401") + + +async def _team_key_rag_query( + general_settings: Mapping[str, object], + key_vector_stores: list[str] | None, + team_vector_stores: list[str] | None, + user_object: LiteLLM_UserTable | None = None, +) -> bool: + return await _common_checks_for_rag_query( + general_settings, + _virtual_key(object_permission_id=None if key_vector_stores is None else "key-permission", team_id="team-1"), + None if key_vector_stores is None else SimpleNamespace(vector_stores=key_vector_stores), + LiteLLM_TeamTable( + team_id="team-1", object_permission_id=None if team_vector_stores is None else "team-permission" + ), + None if team_vector_stores is None else SimpleNamespace(vector_stores=team_vector_stores), + user_object, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("general_settings", "denied_by"), + [ + ({}, None), + ({"vector_store_deny_by_default": False}, None), + ({"vector_store_deny_by_default": True}, _KEY_DENIED), + ], + ids=["flag-omitted", "flag-false", "flag-true"], +) +async def test_standalone_key_without_vector_store_permission_follows_deny_by_default( + general_settings: Mapping[str, object], denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome(_common_checks_for_rag_query(general_settings, _virtual_key(), None), denied_by) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deny_by_default", "vector_stores", "denied_by"), + [ + (True, [], _KEY_DENIED), + (True, ["KBSTOREA"], None), + (True, ["KBSTOREB"], _KEY_DENIED), + (True, None, _KEY_DENIED), + (False, ["KBSTOREB"], _KEY_DENIED), + ], + ids=["enabled-empty", "enabled-contains", "enabled-excludes", "enabled-null", "disabled-excludes"], +) +async def test_standalone_key_vector_store_permission_record_under_deny_by_default( + deny_by_default: bool, vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": deny_by_default}, + _virtual_key(object_permission_id="key-permission"), + SimpleNamespace(vector_stores=vector_stores), + ), + denied_by, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deny_by_default", [False, True], ids=["flag-false", "flag-true"]) +async def test_master_key_rag_query_is_unchanged_by_deny_by_default(deny_by_default: bool): + master_key: Final = _virtual_key(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS, user_role=LitellmUserRoles.PROXY_ADMIN) + + await _assert_rag_query_outcome( + _common_checks_for_rag_query({"vector_store_deny_by_default": deny_by_default}, master_key, None), None + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("general_settings", "vector_stores", "denied_by"), + [ + ({}, None, None), + ({"vector_store_deny_by_default": False}, None, None), + ({"vector_store_deny_by_default": False}, ["KBSTOREB"], _KEY_DENIED), + ], + ids=["flag-omitted-no-record", "flag-false-no-record", "flag-false-excludes"], +) +async def test_proxy_admin_virtual_key_keeps_flag_off_vector_store_behavior( + general_settings: Mapping[str, object], vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None +): + admin_key: Final = _virtual_key( + object_permission_id=None if vector_stores is None else "key-permission", user_role=LitellmUserRoles.PROXY_ADMIN + ) + key_permission: Final = None if vector_stores is None else SimpleNamespace(vector_stores=vector_stores) + + await _assert_rag_query_outcome( + _common_checks_for_rag_query(general_settings, admin_key, key_permission), denied_by + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("key_vector_stores", "team_vector_stores", "denied_by"), + [ + (["KBSTOREA"], ["KBSTOREA"], None), + (None, ["KBSTOREA"], _KEY_DENIED), + ([], ["KBSTOREA"], _KEY_DENIED), + (["KBSTOREA"], None, _TEAM_DENIED), + (["KBSTOREA"], [], _TEAM_DENIED), + (["KBSTOREB"], ["KBSTOREA"], _KEY_DENIED), + (["KBSTOREA"], ["KBSTOREB"], _TEAM_DENIED), + ], + ids=["both-grant", "key-no-record", "key-empty", "team-no-record", "team-empty", "key-excludes", "team-excludes"], +) +async def test_team_key_requires_key_and_team_grant_under_deny_by_default( + key_vector_stores: list[str] | None, team_vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _team_key_rag_query({"vector_store_deny_by_default": True}, key_vector_stores, team_vector_stores), denied_by + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("general_settings", "key_vector_stores", "team_vector_stores", "denied_by"), + [ + ({}, [], ["KBSTOREA"], None), + ({"vector_store_deny_by_default": False}, [], ["KBSTOREA"], None), + ({"vector_store_deny_by_default": False}, None, None, None), + ({"vector_store_deny_by_default": False}, ["KBSTOREB"], ["KBSTOREA"], _KEY_DENIED), + ({"vector_store_deny_by_default": False}, ["KBSTOREA"], ["KBSTOREB"], _TEAM_DENIED), + ], + ids=["omitted-key-empty", "false-key-empty", "false-no-records", "false-key-excludes", "false-team-excludes"], +) +async def test_team_key_keeps_legacy_vector_store_behavior_when_flag_off( + general_settings: Mapping[str, object], + key_vector_stores: list[str] | None, + team_vector_stores: list[str] | None, + denied_by: ProxyErrorTypes | None, +): + await _assert_rag_query_outcome( + _team_key_rag_query(general_settings, key_vector_stores, team_vector_stores), denied_by + ) + + +@pytest.mark.asyncio +async def test_user_owned_team_key_without_key_grant_is_denied_despite_team_membership(): + team_member: Final = LiteLLM_UserTable( + user_id="key-owner", teams=["team-1"], user_role=LitellmUserRoles.INTERNAL_USER + ) + + await _assert_rag_query_outcome( + _team_key_rag_query({"vector_store_deny_by_default": True}, [], ["KBSTOREA"], team_member), _KEY_DENIED + ) + + +def _user_row(user_vector_stores: list[str] | None, teams: tuple[str, ...] = ()) -> LiteLLM_UserTable: + return LiteLLM_UserTable( + user_id="user-1", + teams=list(teams), + user_role=LitellmUserRoles.INTERNAL_USER, + object_permission_id=None if user_vector_stores is None else "user-permission", + ) + + +async def _keyless_rag_query( + general_settings: Mapping[str, object], + user_vector_stores: list[str] | None, + team_vector_stores: list[str] | None = None, + team_id: str | None = None, + user_permission: LiteLLM_ObjectPermissionTable | None = None, +) -> bool: + return await _common_checks_for_rag_query( + general_settings, + UserAPIKeyAuth(user_id="user-1", team_id=team_id, user_role=LitellmUserRoles.INTERNAL_USER), + None, + None + if team_id is None + else LiteLLM_TeamTable( + team_id=team_id, object_permission_id=None if team_vector_stores is None else "team-permission" + ), + None if team_vector_stores is None else SimpleNamespace(vector_stores=team_vector_stores), + _user_row(user_vector_stores, teams=() if team_id is None else (team_id,)), + user_permission or (None if user_vector_stores is None else SimpleNamespace(vector_stores=user_vector_stores)), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("user_vector_stores", "denied_by"), + [(["KBSTOREA"], None), (["KBSTOREB"], _USER_DENIED), ([], _USER_DENIED), (None, _USER_DENIED)], + ids=["user-grants", "user-excludes", "user-empty", "user-no-record"], +) +async def test_keyless_user_without_team_needs_user_grant_under_deny_by_default( + user_vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _keyless_rag_query({"vector_store_deny_by_default": True}, user_vector_stores), denied_by + ) + + +@pytest.mark.asyncio +async def test_keyless_user_null_vector_store_list_grants_nothing_under_deny_by_default(): + null_list: Final = LiteLLM_ObjectPermissionTable(object_permission_id="user-permission", vector_stores=None) + + await _assert_rag_query_outcome( + _keyless_rag_query({"vector_store_deny_by_default": True}, [], user_permission=null_list), _USER_DENIED + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("general_settings", "user_vector_stores"), + [({}, ["KBSTOREB"]), ({"vector_store_deny_by_default": False}, ["KBSTOREB"]), ({}, None)], + ids=["omitted-user-excludes", "false-user-excludes", "omitted-user-no-record"], +) +async def test_keyless_user_keeps_legacy_vector_store_behavior_when_flag_off( + general_settings: Mapping[str, object], user_vector_stores: list[str] | None +): + await _assert_rag_query_outcome(_keyless_rag_query(general_settings, user_vector_stores), None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("team_vector_stores", "user_vector_stores", "denied_by"), + [ + (["KBSTOREA"], [], None), + (["KBSTOREB"], ["KBSTOREA"], _TEAM_DENIED), + ([], ["KBSTOREA"], _TEAM_DENIED), + (None, ["KBSTOREA"], _TEAM_DENIED), + ], + ids=["team-grants-user-empty", "team-excludes-user-grants", "team-empty-user-grants", "team-no-record-user-grants"], +) +async def test_keyless_team_member_needs_only_team_grant_under_deny_by_default( + team_vector_stores: list[str] | None, user_vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _keyless_rag_query({"vector_store_deny_by_default": True}, user_vector_stores, team_vector_stores, "team-1"), + denied_by, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("team_id", "denied_by"), + [("team-1", None), (None, None)], + ids=["session-on-team-uses-team-grant", "session-without-team-uses-user-grant"], +) +async def test_session_token_is_not_a_virtual_key_under_deny_by_default( + team_id: str | None, denied_by: ProxyErrorTypes | None +): + session: Final = UserAPIKeyAuth( + api_key="hashed-session-token", + user_id="user-1", + team_id=team_id, + user_role=LitellmUserRoles.INTERNAL_USER, + is_session_token=True, + ) + session.via_virtual_key = True + + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": True}, + session, + None, + None if team_id is None else LiteLLM_TeamTable(team_id=team_id, object_permission_id="team-permission"), + SimpleNamespace(vector_stores=["KBSTOREA"]), + _user_row(["KBSTOREA"] if team_id is None else []), + SimpleNamespace(vector_stores=["KBSTOREA"] if team_id is None else []), + ), + denied_by, + ) + + +@pytest.mark.asyncio +async def test_user_owned_standalone_key_cannot_use_owner_grants_under_deny_by_default(): + owner: Final = LiteLLM_UserTable( + user_id="key-owner", user_role=LitellmUserRoles.INTERNAL_USER, object_permission_id="user-permission" + ) + + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": True}, + _virtual_key(), + None, + user_object=owner, + user_permission=SimpleNamespace(vector_stores=["KBSTOREA"]), + ), + _KEY_DENIED, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deny_by_default", "key_vector_stores", "denied_by"), + [ + (True, ["KBSTOREA", "KBSTOREB"], None), + (True, ["KBSTOREA"], _KEY_DENIED), + (True, ["KBSTOREB"], _KEY_DENIED), + (False, ["KBSTOREA"], _KEY_DENIED), + ], + ids=["enabled-grants-both", "enabled-missing-second", "enabled-missing-first", "disabled-missing-second"], +) +async def test_every_requested_vector_store_needs_a_grant( + deny_by_default: bool, key_vector_stores: list[str], denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": deny_by_default}, + _virtual_key(object_permission_id="key-permission"), + SimpleNamespace(vector_stores=key_vector_stores), + request_body={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "what is in these KBs?"}], + "tools": [{"type": "file_search", "vector_store_ids": ["KBSTOREB"]}], + "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"}, + }, + vector_store_registry=VectorStoreRegistry(), + ), + denied_by, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "valid_token", + [_virtual_key(), UserAPIKeyAuth(user_id="user-1", user_role=LitellmUserRoles.INTERNAL_USER)], + ids=["standalone-key", "keyless-user"], +) +async def test_request_without_vector_stores_is_unaffected_by_deny_by_default(valid_token: UserAPIKeyAuth): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": True}, + valid_token, + None, + user_object=_user_row(None), + request_body={"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hello"}]}, + vector_store_registry=VectorStoreRegistry(), + ), + None, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_body", + [ + {"query": "what is in this KB?", "vector_store_id": "KBSTOREA", "vector_store_ids": ["KBSTOREA"]}, + { + "model": "gpt-4o-mini", + "input": "what is in this KB?", + "tools": [{"type": "file_search", "vector_store_ids": ["KBSTOREA"]}], + }, + ], + ids=["vector-store-search-route", "responses-file-search"], +) +@pytest.mark.parametrize( + ("deny_by_default", "key_vector_stores", "denied_by"), + [(True, None, _KEY_DENIED), (True, ["KBSTOREA"], None), (False, None, None)], + ids=["enabled-no-grant", "enabled-grant", "disabled-no-grant"], +) +async def test_deny_by_default_reads_requested_vector_stores_without_a_registry( + request_body: Mapping[str, object], + deny_by_default: bool, + key_vector_stores: list[str] | None, + denied_by: ProxyErrorTypes | None, +): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": deny_by_default}, + _virtual_key(object_permission_id=None if key_vector_stores is None else "key-permission"), + None if key_vector_stores is None else SimpleNamespace(vector_stores=key_vector_stores), + request_body=request_body, + ), + denied_by, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_body", "key_vector_stores", "denied_by"), + [ + ({"tools": [1]}, None, None), + ({"tools": 1}, None, None), + ({"tools": [{"type": "file_search", "vector_store_ids": None}]}, None, None), + ({"tools": [1, {"type": "file_search", "vector_store_ids": ["KBSTOREA"]}]}, ["KBSTOREA"], None), + ({"tools": [1, {"type": "file_search", "vector_store_ids": ["KBSTOREA"]}]}, ["KBSTOREB"], _KEY_DENIED), + ], + ids=["int-tool", "non-list-tools", "null-tool-ids", "int-tool-beside-granted", "int-tool-beside-ungranted"], +) +async def test_deny_by_default_ignores_tools_that_name_no_vector_store( + request_body: Mapping[str, object], key_vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": True}, + _virtual_key(object_permission_id=None if key_vector_stores is None else "key-permission"), + None if key_vector_stores is None else SimpleNamespace(vector_stores=key_vector_stores), + request_body={"model": "gpt-4o-mini", "input": "what is in this KB?", **request_body}, + ), + denied_by, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_body", + [ + {"tools": [{"type": "file_search", "vector_store_ids": "KBSTOREA"}]}, + {"tools": [{"type": "file_search", "vector_store_ids": [1]}]}, + {"vector_store_ids": "KBSTOREA"}, + ], + ids=["string-tool-ids", "int-tool-id", "string-top-level-ids"], +) +async def test_deny_by_default_rejects_malformed_vector_store_ids_as_bad_request(request_body: Mapping[str, object]): + with pytest.raises(ProxyException) as exc_info: + await _common_checks_for_rag_query( + {"vector_store_deny_by_default": True}, + _virtual_key(object_permission_id="key-permission"), + SimpleNamespace(vector_stores=["KBSTOREA"]), + request_body={"model": "gpt-4o-mini", "input": "what is in this KB?", **request_body}, + ) + assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == ( + "invalid_request_error", + "vector_store_ids", + "400", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_body", "denied_by"), + [ + ({"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hello"}]}, None), + (None, _KEY_DENIED), + ], + ids=["no-vector-store", "ungranted-vector-store"], +) +@pytest.mark.parametrize("flag_value", [None, "enabled"], ids=["null", "string"]) +async def test_invalid_deny_by_default_value_only_denies_vector_store_requests( + flag_value: object, request_body: Mapping[str, object] | None, denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": flag_value}, _virtual_key(), None, request_body=request_body + ), + denied_by, + ) + + +@pytest.mark.asyncio +async def test_keyless_user_grant_is_read_through_the_object_permission_cache(): + cache: Final = UserApiKeyCache(default_in_memory_ttl=60) + await cache.async_set_cache( + key=object_permission_cache_key("user-permission"), + value=LiteLLM_ObjectPermissionTable(object_permission_id="user-permission", vector_stores=["KBSTOREA"]), + model_type=LiteLLM_ObjectPermissionTable, + ) + + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": True}, + UserAPIKeyAuth(user_id="user-1", user_role=LitellmUserRoles.INTERNAL_USER), + None, + user_object=_user_row(["KBSTOREA"]), + user_api_key_cache=cache, + ), + None, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deny_by_default", [True, False], ids=["strict", "default"]) +async def test_key_and_team_grants_are_read_through_the_object_permission_cache(deny_by_default: bool): + cache: Final = UserApiKeyCache(default_in_memory_ttl=60) + for object_permission_id, vector_stores in (("key-permission", ["KBSTOREA"]), ("team-permission", ["KBSTOREB"])): + await cache.async_set_cache( + key=object_permission_cache_key(object_permission_id), + value=LiteLLM_ObjectPermissionTable(object_permission_id=object_permission_id, vector_stores=vector_stores), + model_type=LiteLLM_ObjectPermissionTable, + ) + + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": deny_by_default}, + _virtual_key(object_permission_id="key-permission", team_id="team-1"), + None, + LiteLLM_TeamTable(team_id="team-1", object_permission_id="team-permission"), + None, + user_api_key_cache=cache, + ), + _TEAM_DENIED, + ) + + def test_can_object_call_model_with_alias(): """Test that can_object_call_model works with model aliases""" from litellm import Router @@ -10346,7 +10936,9 @@ async def test_authoritative_access_group_reads_writer_despite_stale_allow_cache from litellm.proxy._types import LiteLLM_AccessGroupTable from litellm.proxy.auth.auth_checks import get_access_object - stale: Final = LiteLLM_AccessGroupTable(access_group_id="group", access_group_name="Policy", access_model_names=["old"]) + stale: Final = LiteLLM_AccessGroupTable( + access_group_id="group", access_group_name="Policy", access_model_names=["old"] + ) current: Final = stale.model_copy(update={"access_model_names": ["new"] if allowed else []}) client: Final = MagicMock() client.writer_db.litellm_accessgrouptable.find_unique = AsyncMock(return_value=current) @@ -10498,9 +11090,12 @@ async def test_authoritative_key_cannot_keep_grants_when_permission_is_unavailab from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache database: Final = MagicMock() - database.get_data = AsyncMock(return_value=UserAPIKeyAuth( - object_permission_id="grant", object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="grant", agents=["allowed"]) - )) + database.get_data = AsyncMock( + return_value=UserAPIKeyAuth( + object_permission_id="grant", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="grant", agents=["allowed"]), + ) + ) database.writer_db.litellm_objectpermissiontable.find_unique = AsyncMock( return_value=None, side_effect=None if missing else RuntimeError("writer unavailable") ) @@ -10523,7 +11118,9 @@ async def test_authoritative_group_grants_propagate_policy_outages( database: Final = MagicMock() database.db.litellm_accessgrouptable.find_unique = AsyncMock(side_effect=RuntimeError("database unavailable")) - database.writer_db.litellm_accessgrouptable.find_unique = AsyncMock(side_effect=RuntimeError("database unavailable")) + database.writer_db.litellm_accessgrouptable.find_unique = AsyncMock( + side_effect=RuntimeError("database unavailable") + ) monkeypatch.setattr(proxy_server, "prisma_client", database) monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) if strict: diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index d4d11527994..5de1d9300fe 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -12,6 +12,7 @@ from functools import partial from pathlib import Path from textwrap import dedent from types import SimpleNamespace +from typing import Final from unittest.mock import ANY, AsyncMock, MagicMock, patch @@ -21,11 +22,13 @@ from fastapi import HTTPException, status import litellm import litellm.proxy.proxy_server from litellm.caching.dual_cache import DualCache +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy._types import ( LiteLLMRoutes, LiteLLM_JWTAuth, LiteLLM_BudgetTable, LiteLLM_EndUserTable, + LiteLLM_ObjectPermissionTable, LiteLLM_OrganizationTable, LiteLLM_TeamTableCachedObj, LiteLLM_UserTable, @@ -5304,6 +5307,222 @@ async def test_centralized_common_checks_tolerates_db_errors_when_fetching_conte setattr(_proxy_server_mod, k, v) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deny_by_default", "team_lookup_error", "denied"), + [ + (False, RuntimeError("team cache unavailable"), False), + (True, RuntimeError("team cache unavailable"), True), + (True, HTTPException(status_code=404, detail="team read failed"), True), + ], + ids=["flag-off-lookup-swallowed", "flag-on-lookup-swallowed", "flag-on-team-rebuilt-from-token"], +) +async def test_team_key_vector_store_access_when_team_cannot_be_resolved( + monkeypatch: pytest.MonkeyPatch, deny_by_default: bool, team_lookup_error: Exception, denied: bool +): + from fastapi import Request + from starlette.datastructures import URL + + token: Final = UserAPIKeyAuth( + api_key="sk-team-key", team_id="team-1", team_models=["gpt-4o-mini"], object_permission_id="key-permission" + ) + token.via_virtual_key = True + request: Final = Request(scope={"type": "http"}) + request._url = URL(url="/v1/rag/query") + database: Final = MagicMock() + database.db.litellm_objectpermissiontable.find_unique = AsyncMock( + side_effect=lambda where: LiteLLM_ObjectPermissionTable( + object_permission_id=where["object_permission_id"], vector_stores=["KBSTOREA"] + ) + if where["object_permission_id"] == "key-permission" + else None + ) + attrs: Final = { + **_proxy_attrs_for_centralized_checks(), + "prisma_client": database, + "proxy_logging_obj": MagicMock(service_logging_obj=MagicMock(async_service_success_hook=AsyncMock())), + "general_settings": {"vector_store_deny_by_default": deny_by_default}, + } + for name, value in attrs.items(): + monkeypatch.setattr(litellm.proxy.proxy_server, name, value) + monkeypatch.setattr(litellm, "vector_store_registry", None) + monkeypatch.setattr( + "litellm.proxy.auth.user_api_key_auth.get_team_object", AsyncMock(side_effect=team_lookup_error) + ) + + checks: Final = _run_centralized_common_checks( + user_api_key_auth_obj=token, + request=request, + request_data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "what is in this KB?"}], + "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"}, + }, + route="/v1/rag/query", + ) + if not denied: + await checks + return + with pytest.raises(ProxyException) as exc_info: + await checks + assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == ( + ProxyErrorTypes.team_vector_store_access_denied, + "vector_store", + "401", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("team_lookup", "denied"), + [ + ({"team-1": "team-1-grants-a"}, False), + (RuntimeError("team cache unavailable"), True), + ({"team-1": "team-1-grants-none", "team-2": "team-2-grants-a"}, True), + ], + ids=["resolved-team-grants", "team-lookup-swallowed-no-personal-fallback", "other-member-team-grants-ignored"], +) +async def test_keyless_team_member_vector_store_access_uses_only_the_resolved_team( + monkeypatch: pytest.MonkeyPatch, team_lookup: dict[str, str] | Exception, denied: bool +): + from fastapi import Request + from starlette.datastructures import URL + + token: Final = UserAPIKeyAuth( + user_id="user-1", + team_id="team-1", + team_models=["gpt-4o-mini"], + user_role=LitellmUserRoles.INTERNAL_USER, + ) + request: Final = Request(scope={"type": "http"}) + request._url = URL(url="/v1/rag/query") + grants: Final = { + "team-1-grants-a": ["KBSTOREA"], + "team-1-grants-none": [], + "team-2-grants-a": ["KBSTOREA"], + "user-grants-a": ["KBSTOREA"], + } + database: Final = MagicMock() + database.db.litellm_objectpermissiontable.find_unique = AsyncMock( + side_effect=lambda where: LiteLLM_ObjectPermissionTable( + object_permission_id=where["object_permission_id"], vector_stores=grants[where["object_permission_id"]] + ) + ) + attrs: Final = { + **_proxy_attrs_for_centralized_checks(), + "prisma_client": database, + "proxy_logging_obj": MagicMock(service_logging_obj=MagicMock(async_service_success_hook=AsyncMock())), + "general_settings": {"vector_store_deny_by_default": True}, + } + for name, value in attrs.items(): + monkeypatch.setattr(litellm.proxy.proxy_server, name, value) + monkeypatch.setattr(litellm, "vector_store_registry", None) + + async def get_team(team_id: str, **_: object) -> LiteLLM_TeamTableCachedObj: + if isinstance(team_lookup, Exception): + raise team_lookup + return LiteLLM_TeamTableCachedObj(team_id=team_id, models=["gpt-4o-mini"], object_permission_id=team_lookup[team_id]) + + monkeypatch.setattr("litellm.proxy.auth.user_api_key_auth.get_team_object", get_team) + monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr( + "litellm.proxy.auth.user_api_key_auth.get_user_object", + AsyncMock( + return_value=LiteLLM_UserTable( + user_id="user-1", teams=["team-1", "team-2"], object_permission_id="user-grants-a" + ) + ), + ) + + checks: Final = _run_centralized_common_checks( + user_api_key_auth_obj=token, + request=request, + request_data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "what is in this KB?"}], + "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"}, + }, + route="/v1/rag/query", + ) + if not denied: + await checks + return + with pytest.raises(ProxyException) as exc_info: + await checks + assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == ( + ProxyErrorTypes.team_vector_store_access_denied, + "vector_store", + "401", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("user_permission_id", "denied"), + [("admin-grants-a", False), (None, True)], + ids=["admin-personal-grant", "admin-without-grant"], +) +async def test_keyless_proxy_admin_keeps_personal_vector_store_grants_under_deny_by_default( + monkeypatch: pytest.MonkeyPatch, user_permission_id: str | None, denied: bool +): + from fastapi import Request + from starlette.datastructures import URL + + token: Final = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + request: Final = Request(scope={"type": "http"}) + request._url = URL(url="/v1/rag/query") + database: Final = MagicMock() + database.db.litellm_objectpermissiontable.find_unique = AsyncMock( + side_effect=lambda where: SimpleNamespace( + dict=lambda: {"object_permission_id": where["object_permission_id"], "vector_stores": ["KBSTOREA"]}, + vector_stores=["KBSTOREA"], + ) + ) + attrs: Final = { + **_proxy_attrs_for_centralized_checks(), + "prisma_client": database, + "general_settings": {"vector_store_deny_by_default": True}, + "user_api_key_cache": UserApiKeyCache(), + "proxy_logging_obj": MagicMock( + service_logging_obj=MagicMock( + async_service_success_hook=AsyncMock(), async_service_failure_hook=AsyncMock() + ) + ), + } + for name, value in attrs.items(): + monkeypatch.setattr(litellm.proxy.proxy_server, name, value) + monkeypatch.setattr(litellm, "vector_store_registry", None) + monkeypatch.setattr( + "litellm.proxy.auth.user_api_key_auth.get_user_object", + AsyncMock( + return_value=LiteLLM_UserTable( + user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN, object_permission_id=user_permission_id + ) + ), + ) + + checks: Final = _run_centralized_common_checks( + user_api_key_auth_obj=token, + request=request, + request_data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "what is in this KB?"}], + "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"}, + }, + route="/v1/rag/query", + ) + if not denied: + await checks + return + with pytest.raises(ProxyException) as exc_info: + await checks + assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == ( + ProxyErrorTypes.user_vector_store_access_denied, + "vector_store", + "401", + ) + + @pytest.mark.asyncio async def test_centralized_common_checks_propagates_end_user_budget_error(): """Regression: ``get_end_user_object`` raises ``litellm.BudgetExceededError`` diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index 7150ba892df..912622da4bd 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -4066,6 +4066,34 @@ async def test_user_update_invalidates_the_cached_entitlement(mocker): } +@pytest.mark.asyncio +async def test_user_update_broadcasts_the_entitlement_invalidation_to_other_workers(mocker: MockerFixture): + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + _object_permission_mocks(mocker) + cache: Final = mocker.MagicMock() + cache.async_delete_cache = mocker.AsyncMock() + mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache) # test-quality-ok: substitute the cache dependency + broadcast: Final = mocker.patch( # test-quality-ok: observe the Redis publication boundary + "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.publish_auth_cache_invalidation", + new_callable=mocker.AsyncMock, + ) + + await _update_single_user_helper( + user_request=UpdateUserRequest(user_id="target-user", object_permission={"vector_stores": []}), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + broadcast_keys: Final = {call.kwargs["cache_key"] for call in broadcast.await_args_list} + assert broadcast_keys == { + "object_permission_id:perm-new", + "user_object_permission_id:target-user", + "target-user", + } + + @pytest.mark.asyncio async def test_admin_can_clear_a_users_mcp_entitlement(mocker): """An explicit empty object_permission means "no object permission", so it must unlink. diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index 6f6dbe1e954..6c5447fce64 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -8417,6 +8417,7 @@ async def test_update_team_org_scoped_tpm_rpm_bypasses_user_limit( # Mock team update mock_updated_team = MagicMock(spec=LiteLLM_TeamTable) mock_updated_team.team_id = "org-team-update-bypass-123" + mock_updated_team.object_permission_id = None mock_updated_team.tpm_limit = 10000 mock_updated_team.rpm_limit = 1000 mock_updated_team.access_group_ids = None @@ -8570,6 +8571,7 @@ async def test_update_team_guardrails_with_org_id( # Mock team update mock_updated_team = MagicMock(spec=LiteLLM_TeamTable) mock_updated_team.team_id = "team-guardrails-123" + mock_updated_team.object_permission_id = None mock_updated_team.organization_id = "test-org-guardrails" mock_updated_team.metadata = { "guardrails": ["aporia-pre-call", "aporia-post-call"] diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 2f5eb81bec8..b37576dab80 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -26,7 +26,7 @@ import pytest from pydantic import JsonValue, TypeAdapter, ValidationError import litellm -from litellm.proxy._types import CommonProxyErrors +from litellm.proxy._types import CommonProxyErrors, ConfigGeneralSettings from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.proxy_server import ( ProxyConfig, @@ -2329,6 +2329,42 @@ async def test_load_config_logs_disabled_budget_reservation_once(tmp_path, monke assert [record.levelno for record in records] == ([logging.INFO] if setting == "true" else []) +@pytest.mark.asyncio +@pytest.mark.parametrize(("yaml_value", "expected"), [("true", True), ("false", False)]) +async def test_load_config_yaml_vector_store_deny_by_default_is_boolean( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, yaml_value: str, expected: bool +): + config_file: Final = tmp_path / "vector_store.yaml" + config_file.write_text( + f"model_list: []\nlitellm_settings: {{}}\ngeneral_settings:\n vector_store_deny_by_default: {yaml_value}\n" + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + + _, _, general_settings = await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) + + assert general_settings["vector_store_deny_by_default"] is expected + assert ConfigGeneralSettings.model_validate(dict(general_settings)).vector_store_deny_by_default is expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("yaml_value", ["", "enabled"], ids=["null", "string"]) +async def test_load_config_rejects_non_boolean_vector_store_deny_by_default( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, yaml_value: str +): + config_file: Final = tmp_path / "vector_store.yaml" + config_file.write_text( + f"model_list: []\nlitellm_settings: {{}}\ngeneral_settings:\n vector_store_deny_by_default: {yaml_value}\n" + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + + with pytest.raises(ValidationError, match="vector_store_deny_by_default"): + await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) + + @pytest.mark.asyncio async def test_ProxyConfig_load_config_resolves_router_settings_plugins(tmp_path, monkeypatch): """Regression: router_settings.plugins dotted-path strings must be resolved to diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c5b8c0e4d14..c632ad01203 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30083,6 +30083,12 @@ export interface components { * @description Master switch for the SSRF guard applied to user-supplied URLs (image_url, file_url, MCP/OpenAPI spec URLs, etc). Defaults to True. Set to False to disable DNS/IP validation entirely (not recommended). */ user_url_validation?: boolean | null; + /** + * Vector Store Deny By Default + * @description When True, a vector store must be explicitly listed in object_permission.vector_stores: a virtual key needs its own grant plus its team's, a keyless team member needs the team's, and a user with neither needs their own. A missing permission record, an empty list, or an unresolved team grants nothing. Dashboard session keys are not yet covered + * @default false + */ + vector_store_deny_by_default: boolean; }; /** ConfigList */ ConfigList: { From 282d733fb6515f4dd2acb5b86ad57f6d2f37f075 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:50:26 -0700 Subject: [PATCH 3/5] feat(auth): deny search tools by default when search_tool_deny_by_default is set (#44490) Co-authored-by: mrinal Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../websearch_interception/handler.py | 45 +-- .../litellm_core_utils/error_normalization.py | 3 + litellm/proxy/_types.py | 29 ++ litellm/proxy/auth/auth_checks.py | 246 ++++++++++---- litellm/proxy/auth/fallback_model_access.py | 35 +- litellm/proxy/proxy_server.py | 13 +- litellm/proxy/search_endpoints/endpoints.py | 52 +-- .../search_tool_management.py | 41 +-- .../test_search_tool_deny_by_default.py | 221 +++++++++++++ .../test_websearch_interception_handler.py | 189 ++++++++++- .../proxy/auth/test_fallback_model_access.py | 69 +++- .../auth/test_search_tool_deny_by_default.py | 270 ++++++++++++++++ .../test_search_tool_management.py | 300 ++++++++++++------ .../proxy/proxy_server/test_proxy_config.py | 32 +- .../proxy/search_endpoints/test_endpoints.py | 235 +++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 + 16 files changed, 1497 insertions(+), 289 deletions(-) create mode 100644 tests/integration/authorization/test_search_tool_deny_by_default.py create mode 100644 tests/unit/proxy/auth/test_search_tool_deny_by_default.py diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 1db94e82066..15413140d9e 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -1557,8 +1557,8 @@ class WebSearchInterceptionLogger(CustomLogger): search_litellm_params = dict[str, object](tool_params) search_provider = tool_params.get("search_provider") - # Fallback to perplexity if no router or no search tools configured if not search_provider: + self._authorize_unregistered_search_fallback(kwargs=kwargs) search_provider = "perplexity" verbose_logger.debug( "WebSearchInterception: No search tools configured in router, using default provider '%s'", @@ -1623,6 +1623,18 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.error("WebSearchInterception: Search failed for '%s': %s", query, e) raise + def _authorize_unregistered_search_fallback(self, kwargs: Mapping[str, object] | None) -> None: + user_api_key_auth: Final = self._get_user_api_key_auth_from_kwargs(kwargs) + if user_api_key_auth is None: + return + + from litellm.proxy.auth.auth_checks import check_unregistered_search_fallback, typed_general_settings + from litellm.proxy.proxy_server import general_settings + + check_unregistered_search_fallback( + valid_token=user_api_key_auth, general_settings=typed_general_settings(general_settings) + ) + async def _authorize_search_tool( self, search_tool: Mapping[str, object], @@ -1636,36 +1648,9 @@ class WebSearchInterceptionLogger(CustomLogger): if user_api_key_auth is None: return - from litellm.proxy.auth.auth_checks import ( - can_key_call_search_tool, - can_team_call_search_tool, - get_team_object, - ) + from litellm.proxy.auth.auth_checks import can_token_call_search_tool - await can_key_call_search_tool( - search_tool_name=search_tool_name, - valid_token=user_api_key_auth, - ) - - team_id: Final[str | None] = getattr(user_api_key_auth, "team_id", None) - if team_id: - from litellm.proxy.proxy_server import ( - prisma_client, - proxy_logging_obj, - user_api_key_cache, - ) - - team_object: Final = await get_team_object( - team_id=team_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=getattr(user_api_key_auth, "parent_otel_span", None), - proxy_logging_obj=proxy_logging_obj, - ) - await can_team_call_search_tool( - search_tool_name=search_tool_name, - team_object=team_object, - ) + await can_token_call_search_tool(search_tool_name=search_tool_name, valid_token=user_api_key_auth) @staticmethod def _build_search_request_metadata( diff --git a/litellm/litellm_core_utils/error_normalization.py b/litellm/litellm_core_utils/error_normalization.py index 4426d42f767..00317e502ec 100644 --- a/litellm/litellm_core_utils/error_normalization.py +++ b/litellm/litellm_core_utils/error_normalization.py @@ -108,6 +108,9 @@ _PROXY_ERROR_TYPE_MAP: Final[Mapping[str, str]] = MappingProxyType( "team_vector_store_access_denied": PERMISSION_DENIED, "org_vector_store_access_denied": PERMISSION_DENIED, "user_vector_store_access_denied": PERMISSION_DENIED, + "key_search_tool_access_denied": PERMISSION_DENIED, + "team_search_tool_access_denied": PERMISSION_DENIED, + "user_search_tool_access_denied": PERMISSION_DENIED, "tool_access_denied": PERMISSION_DENIED, "team_member_permission_error": PERMISSION_DENIED, "not_found_error": RESOURCE_NOT_FOUND, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 23cd64b8a49..d02d1f29ded 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3026,6 +3026,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): default=False, description="When True, a vector store must be explicitly listed in object_permission.vector_stores: a virtual key needs its own grant plus its team's, a keyless team member needs the team's, and a user with neither needs their own. A missing permission record, an empty list, or an unresolved team grants nothing. Dashboard session keys are not yet covered", ) + search_tool_deny_by_default: bool = Field( + default=False, + description="When True, a search tool must be explicitly listed in object_permission.search_tools: a virtual key needs its own grant plus its team's, a keyless team member needs the team's, and a user with neither needs their own. A missing permission record, an empty list, or an unresolved team grants nothing, and the unregistered search fallback is denied. The master key and dashboard sessions are exempt", + ) missing_session_id: Literal["generate", "reject", "omit"] | None = Field( None, description="What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400; 'omit' leaves SpendLogs.session_id null, matching callbacks such as Langfuse that only record a client-established metadata.session_id. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.", @@ -4521,6 +4525,21 @@ class ProxyErrorTypes(str, enum.Enum): User does not have access to the vector store """ + key_search_tool_access_denied = "key_search_tool_access_denied" + """ + Key does not have access to the search tool + """ + + team_search_tool_access_denied = "team_search_tool_access_denied" + """ + Team does not have access to the search tool + """ + + user_search_tool_access_denied = "user_search_tool_access_denied" + """ + User does not have access to the search tool + """ + team_member_already_in_team = "team_member_already_in_team" """ Team member is already in team @@ -4569,6 +4588,16 @@ class ProxyErrorTypes(str, enum.Enum): elif object_type == "user": return cls.user_vector_store_access_denied + @classmethod + def get_search_tool_access_error_type_for_object( + cls, object_type: Literal["key", "team", "user"] + ) -> "ProxyErrorTypes": + return { + "key": cls.key_search_tool_access_denied, + "team": cls.team_search_tool_access_denied, + "user": cls.user_search_tool_access_denied, + }[object_type] + DB_CONNECTION_ERROR_TYPES: Final = ( httpx.ConnectError, diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 3ebd1a2cfe0..33fcac6230a 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -14,6 +14,7 @@ import math import re import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass from functools import partial from itertools import chain from types import MappingProxyType @@ -312,6 +313,9 @@ def _user_table(repo: _PrismaTableHolder[_PrismaUserRow]) -> _PrismaAuthTable[_P return _DeadlineBoundedTable(repo.table, "user") +GrantLayer = Literal["key", "team", "user"] + + class _VectorStorePermissionsRow(Protocol): @property def vector_stores(self) -> Sequence[str] | None: ... @@ -386,6 +390,9 @@ def _typed_request_body(request_body: dict) -> Mapping[str, object]: return request_body +typed_general_settings: Final = _typed_request_body + + class _JsonLoadsObj(Protocol): def __call__(self, data: str) -> object: ... @@ -5481,10 +5488,10 @@ def _search_tool_names_from_object_permission( def _can_object_call_search_tool( search_tool_name: str, allowed_search_tools: list[str], - object_type: Literal["key", "team", "project"], + object_type: Literal["key", "team", "project", "user"], ) -> Literal[True]: """ - Check if an object (key/team/project) can access a specific search tool. + Check if an object (key/team/project/user) can access a specific search tool. Similar to _can_object_call_model but for search tools. @@ -5517,84 +5524,188 @@ def _can_object_call_search_tool( ) -async def can_key_call_search_tool( - search_tool_name: str, - valid_token: UserAPIKeyAuth, -) -> Literal[True]: +TeamObjectLoader: TypeAlias = Callable[[], Awaitable[LiteLLM_TeamTable | None]] + + +@dataclass(frozen=True) +class SearchToolGrants: """ - Check if a key can access a specific search tool. - - Similar to can_key_call_model but for search tools. - - Args: - search_tool_name: The search tool being requested - valid_token: The authenticated key - - Returns: - True if access is allowed - - Raises: - ProxyException if access is denied + The key, team and user grants that scope one caller's search tools. Every layer in `strict_layers` + must list the tool, and any other loaded grant only narrows when its list is nonempty """ - return _can_object_call_search_tool( - search_tool_name=search_tool_name, - allowed_search_tools=_search_tool_names_from_object_permission(valid_token.object_permission), - object_type="key", - ) + + grants: Mapping[GrantLayer, LiteLLM_ObjectPermissionTable | None] + strict_layers: frozenset[GrantLayer] -async def can_team_call_search_tool( - search_tool_name: str, - team_object: LiteLLM_TeamTable | None, -) -> Literal[True]: +def _search_tool_deny_by_default(general_settings: Mapping[str, object]) -> bool: """ - Check if a team can access a specific search tool. - - Similar to can_team_access_model but for search tools. - - Args: - search_tool_name: The search tool being requested - team_object: The team object - - Returns: - True if access is allowed - - Raises: - ProxyException if access is denied - """ - if team_object is None: - return True - - return _can_object_call_search_tool( - search_tool_name=search_tool_name, - allowed_search_tools=_search_tool_names_from_object_permission(team_object.object_permission), - object_type="team", - ) - - -async def can_user_view_search_tool( - search_tool_name: str, - valid_token: UserAPIKeyAuth, - team_object: LiteLLM_TeamTable | None, -) -> bool: - """ - Boolean variant of the key + team authorization enforced on /search, used to - scope /search_tools/list so a non-admin caller only sees tools it may invoke. + Startup rejects a non-boolean value from the config file. A non-boolean value that reaches + general_settings another way enables the policy, so only search tool requests are denied. """ try: - await can_key_call_search_tool( - search_tool_name=search_tool_name, - valid_token=valid_token, + return ConfigGeneralSettings.model_validate( + MappingProxyType( + {"search_tool_deny_by_default": general_settings.get("search_tool_deny_by_default", False)} + ) + ).search_tool_deny_by_default + except ValidationError: + return True + + +def is_search_tool_deny_by_default_applied(valid_token: UserAPIKeyAuth, general_settings: Mapping[str, object]) -> bool: + return _search_tool_deny_by_default(general_settings) and _is_strict_grant_identity(valid_token) + + +def _search_tool_denied(object_type: GrantLayer, message: str) -> ProxyException: + return ProxyException( + message=message, + type=ProxyErrorTypes.get_search_tool_access_error_type_for_object(object_type), + param="search_tool_name", + code=status.HTTP_403_FORBIDDEN, + ) + + +async def _strict_team_object(load_team_object: TeamObjectLoader) -> LiteLLM_TeamTable | None: + try: + return await load_team_object() + except Exception as e: # noqa: BLE001 # an unresolved team grants nothing under deny-by-default + verbose_proxy_logger.debug("Team lookup failed under search_tool_deny_by_default: %s", e) + return None + + +async def _strict_user_object( + valid_token: UserAPIKeyAuth, prisma_client: PrismaClient, user_api_key_cache: UserApiKeyCache +) -> LiteLLM_UserTable | None: + if valid_token.user_id is None: + return None + try: + return await get_user_object( + user_id=valid_token.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=valid_token.parent_otel_span, ) - await can_team_call_search_tool( - search_tool_name=search_tool_name, - team_object=team_object, + except Exception as e: # noqa: BLE001 # an unresolved user grants nothing under deny-by-default + verbose_proxy_logger.debug("User lookup failed under search_tool_deny_by_default: %s", e) + return None + + +async def resolve_search_tool_grants( + valid_token: UserAPIKeyAuth, + general_settings: Mapping[str, object], + load_team_object: TeamObjectLoader, +) -> SearchToolGrants: + """ + Without `general_settings.search_tool_deny_by_default`, or for the master key and dashboard sessions, + the key and team lists only restrict when nonempty. With it, the identities `_strict_grant_layers` names + must each list the tool, and a missing record, `null`, `[]`, or a team or user that fails to load grants nothing + """ + if not is_search_tool_deny_by_default_applied(valid_token, general_settings): + team_object: Final = await load_team_object() + return SearchToolGrants( + grants=MappingProxyType( + {"key": valid_token.object_permission, "team": team_object.object_permission if team_object else None} + ), + strict_layers=frozenset(), ) + + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + + strict_team_object: Final = await _strict_team_object(load_team_object) + strict_layers: Final = _strict_grant_layers(True, valid_token, strict_team_object) + if prisma_client is None: + return SearchToolGrants(grants=MappingProxyType({}), strict_layers=strict_layers) + user_object: Final = ( + await _strict_user_object(valid_token, prisma_client, user_api_key_cache) if "user" in strict_layers else None + ) + return SearchToolGrants( + grants=MappingProxyType( + dict( + await _identity_grants(valid_token, strict_team_object, user_object, prisma_client, user_api_key_cache) + ) + ), + strict_layers=strict_layers, + ) + + +def check_search_tool_grants(search_tool_name: str, grants: SearchToolGrants) -> Literal[True]: + """Raises a 403 ProxyException naming the first key, team or user layer that does not grant the tool""" + for layer in ("key", "team", "user"): + grant = grants.grants.get(layer) + if layer in grants.strict_layers: + if grant is None or search_tool_name not in (grant.search_tools or ()): + raise _search_tool_denied( + layer, + f"{layer.capitalize()} not allowed to access search tool: {search_tool_name}. " + f"search_tool_deny_by_default is enabled and the {layer} does not grant it", + ) + elif grant is not None: + _can_object_call_search_tool( + search_tool_name=search_tool_name, + allowed_search_tools=_search_tool_names_from_object_permission(grant), + object_type=layer, + ) + return True + + +async def can_caller_call_search_tool( + search_tool_name: str, + valid_token: UserAPIKeyAuth, + general_settings: Mapping[str, object], + load_team_object: TeamObjectLoader, +) -> Literal[True]: + """Key, team and user search tool authorization shared by /search, web search interception and discovery""" + return check_search_tool_grants( + search_tool_name, await resolve_search_tool_grants(valid_token, general_settings, load_team_object) + ) + + +async def can_token_call_search_tool(search_tool_name: str, valid_token: UserAPIKeyAuth) -> Literal[True]: + """`can_caller_call_search_tool` against the proxy's own settings, team cache and database""" + from litellm.proxy.proxy_server import general_settings, prisma_client, proxy_logging_obj, user_api_key_cache + + async def _load_team_object() -> LiteLLM_TeamTable | None: + if not valid_token.team_id: + return None + return await get_team_object( + team_id=valid_token.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=valid_token.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + + return await can_caller_call_search_tool( + search_tool_name=search_tool_name, + valid_token=valid_token, + general_settings=typed_general_settings(general_settings), + load_team_object=_load_team_object, + ) + + +def can_grants_view_search_tool(search_tool_name: str, grants: SearchToolGrants) -> bool: + try: + check_search_tool_grants(search_tool_name, grants) except ProxyException: return False return True +def check_unregistered_search_fallback( + valid_token: UserAPIKeyAuth, general_settings: Mapping[str, object] +) -> Literal[True]: + """The unregistered provider fallback names no search tool that a grant could list, so deny-by-default denies it""" + if not is_search_tool_deny_by_default_applied(valid_token, general_settings): + return True + layer: Final = min(_strict_grant_layers(True, valid_token, None), key=("key", "team", "user").index) + raise _search_tool_denied( + layer, + "No registered search tool is available and search_tool_deny_by_default is enabled", + ) + + async def is_valid_fallback_model( model: str, llm_router: Router | None, @@ -6792,9 +6903,6 @@ def _get_rag_query_vector_store_id(request_body: Mapping[str, object]) -> str | return vector_store_id if isinstance(vector_store_id, str) and vector_store_id else None -GrantLayer = Literal["key", "team", "user"] - - def _is_strict_grant_identity(valid_token: UserAPIKeyAuth | None) -> bool: return ( valid_token is not None diff --git a/litellm/proxy/auth/fallback_model_access.py b/litellm/proxy/auth/fallback_model_access.py index 53807f076ea..39346c80b22 100644 --- a/litellm/proxy/auth/fallback_model_access.py +++ b/litellm/proxy/auth/fallback_model_access.py @@ -5,7 +5,8 @@ Authorize router fallback targets against the caller's key, team and project mod Fallbacks configured on the router (`router_settings.fallbacks` and friends) are chosen after auth, inside the router, so this predicate is injected into the router to re-run the same model access checks for each fallback target before it is attempted. Opt-in via -`general_settings.enforce_fallback_model_access: true`. +`general_settings.enforce_fallback_model_access: true`. Fallbacks of a search request always run the +key, team and user search tool grants instead, the same check /search runs on the requested tool. """ from collections.abc import Callable, Mapping @@ -16,8 +17,9 @@ from pydantic import ValidationError from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ProxyException, UserAPIKeyAuth -from litellm.proxy.auth.auth_checks import can_key_call_resolved_model +from litellm.proxy.auth.auth_checks import can_key_call_resolved_model, can_token_call_search_tool from litellm.router import Router +from litellm.search import asearch from litellm.types.llms.base import LiteLLMBaseModel @@ -45,6 +47,23 @@ async def is_model_authorized_for_token(*, model: str, valid_token: UserAPIKeyAu return True +async def is_search_tool_authorized_for_token(*, search_tool_name: str, valid_token: UserAPIKeyAuth) -> bool: + try: + await can_token_call_search_tool(search_tool_name=search_tool_name, valid_token=valid_token) + except ProxyException: + return False + except Exception as e: # noqa: BLE001 # fail closed: a lookup failure must neither run the fallback nor replace the provider error + verbose_proxy_logger.warning( + "Skipping fallback to search tool=%s: authorization lookup failed: %s", search_tool_name, e + ) + return False + return True + + +def _is_search_request(request_kwargs: Mapping[str, object]) -> bool: + return request_kwargs.get("original_generic_function") is asearch + + def _token_in_metadata(metadata: object) -> UserAPIKeyAuth | None: try: return _RequestMetadata.model_validate(metadata).user_api_key_auth @@ -72,19 +91,21 @@ def _enforced_by_general_settings() -> bool: @dataclass(frozen=True, slots=True) class RouterFallbackAccessCheck: """ - `FallbackAccessCheck` for the proxy's router: while `is_enforced()` is true, a fallback target - is attempted only when the key behind the request could have requested it directly. Requests - that carry no key (for example internal health checks) are not restricted. + `FallbackAccessCheck` for the proxy's router: a fallback of a search request, and while `is_enforced()` + is true any other fallback, is attempted only when the key behind the request could have requested it + directly. Requests that carry no key (for example internal health checks) are not restricted. """ is_enforced: Callable[[], bool] async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool: - if not self.is_enforced(): - return True valid_token: Final = _user_api_key_auth_from_request(request_kwargs) if valid_token is None: return True + if _is_search_request(request_kwargs): + return await is_search_tool_authorized_for_token(search_tool_name=model, valid_token=valid_token) + if not self.is_enforced(): + return True return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=llm_router) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6d522759eb7..5d6ba0f8e13 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6714,12 +6714,15 @@ class ProxyConfig: general_settings = {} typed_general_settings: Final = _GENERAL_SETTINGS_VIEW.validate_python(general_settings) - if "vector_store_deny_by_default" in typed_general_settings: - ConfigGeneralSettings.model_validate( - MappingProxyType( - {"vector_store_deny_by_default": typed_general_settings["vector_store_deny_by_default"]} - ) + ConfigGeneralSettings.model_validate( + MappingProxyType( + { + name: typed_general_settings[name] + for name in ("vector_store_deny_by_default", "search_tool_deny_by_default") + if name in typed_general_settings + } ) + ) if general_settings.get("mcp_advertised_versions") is not None: from litellm.types.mcp import MCPAdvertisedVersions diff --git a/litellm/proxy/search_endpoints/endpoints.py b/litellm/proxy/search_endpoints/endpoints.py index 349eb5b7861..1e37e3243ea 100644 --- a/litellm/proxy/search_endpoints/endpoints.py +++ b/litellm/proxy/search_endpoints/endpoints.py @@ -135,52 +135,24 @@ async def search( if search_tool_name is not None: data["search_tool_name"] = search_tool_name - if not ( - data.get("search_tool_name") or data.get("model") or general_settings.get("completion_model") or user_model - ): + from litellm.proxy.auth.auth_checks import can_token_call_search_tool + from litellm.proxy.common_utils.http_parsing_utils import resolve_inference_model + + routed_search_tool_name: Final = data.get("search_tool_name") or resolve_inference_model( + data.get("model"), general_settings, user_model + ) + if not isinstance(routed_search_tool_name, str) or not routed_search_tool_name: raise ProxyMissingRequiredParamError(route="/search", param="search_tool_name") + try: + await can_token_call_search_tool(search_tool_name=routed_search_tool_name, valid_token=user_api_key_dict) + except ProxyException as e: + verbose_proxy_logger.debug("Search tool authorization denied: %s", e.type) + raise if "search_tool_name" in data and data["search_tool_name"]: data["model"] = data["search_tool_name"] search_tool_name_value: Final = data["search_tool_name"] - # Authorization check: verify key can access this search tool - from litellm.proxy.auth.auth_checks import ( - can_key_call_search_tool, - can_team_call_search_tool, - get_team_object, - ) - - try: - # Check key-level access - await can_key_call_search_tool( - search_tool_name=search_tool_name_value, - valid_token=user_api_key_dict, - ) - - # Check team-level access if key is associated with a team - if user_api_key_dict.team_id: - from litellm.proxy.proxy_server import ( - prisma_client, - proxy_logging_obj, - user_api_key_cache, - ) - - team_object: Final = await get_team_object( - team_id=user_api_key_dict.team_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=user_api_key_dict.parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) - await can_team_call_search_tool( - search_tool_name=search_tool_name_value, - team_object=team_object, - ) - except Exception as e: - verbose_proxy_logger.error("Search tool authorization failed for %s: %s", search_tool_name_value, e) - raise - if llm_router is not None and hasattr(llm_router, "search_tools"): verbose_proxy_logger.debug( "Search endpoint - Looking for search_tool_name: %s. Available search tools in router: %s. Total search tools: %s", diff --git a/litellm/proxy/search_endpoints/search_tool_management.py b/litellm/proxy/search_endpoints/search_tool_management.py index 49b868ac71a..1e6ed40bba3 100644 --- a/litellm/proxy/search_endpoints/search_tool_management.py +++ b/litellm/proxy/search_endpoints/search_tool_management.py @@ -2,7 +2,7 @@ CRUD ENDPOINTS FOR SEARCH TOOLS """ -from collections.abc import Awaitable, Callable, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from datetime import datetime from typing import Any, Final, TypeAlias @@ -117,34 +117,39 @@ async def _filter_visible_search_tools( search_tools: list[SearchToolInfoResponse], user_api_key_dict: UserAPIKeyAuth, lookup_team_object: TeamObjectLookup = _team_object_from_db, + general_settings: Mapping[str, object] | None = None, ) -> list[SearchToolInfoResponse]: """ Drop search tools the caller is not authorized to invoke, applying the same - key/team object_permission allowlists enforced on /search. Admins see all tools. + key/team/user grants enforced on /search. Admins see all tools unless + search_tool_deny_by_default applies to their credential. """ + from litellm.proxy.auth.auth_checks import ( + can_grants_view_search_tool, + is_search_tool_deny_by_default_applied, + resolve_search_tool_grants, + typed_general_settings, + ) + from litellm.proxy.proxy_server import general_settings as proxy_general_settings + + settings: Final = typed_general_settings(proxy_general_settings) if general_settings is None else general_settings if user_api_key_dict.user_role in ( LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - ): + ) and not is_search_tool_deny_by_default_applied(user_api_key_dict, settings): return search_tools - from litellm.proxy.auth.auth_checks import can_user_view_search_tool - allowlist_team_id: Final = _allowlist_team_id(user_api_key_dict) - team_object: Final[LiteLLM_TeamTable | None] = ( - await lookup_team_object(allowlist_team_id, user_api_key_dict) if allowlist_team_id else None - ) - visible: Final[list[SearchToolInfoResponse]] = [] - for tool in search_tools: - tool_name = tool.get("search_tool_name") - if tool_name and await can_user_view_search_tool( - search_tool_name=tool_name, - valid_token=user_api_key_dict, - team_object=team_object, - ): - visible.append(tool) - return visible + async def _load_team_object() -> LiteLLM_TeamTable | None: + return await lookup_team_object(allowlist_team_id, user_api_key_dict) if allowlist_team_id else None + + grants: Final = await resolve_search_tool_grants(user_api_key_dict, settings, _load_team_object) + return [ + tool + for tool in search_tools + if (tool_name := tool.get("search_tool_name")) and can_grants_view_search_tool(tool_name, grants) + ] @router.get( diff --git a/tests/integration/authorization/test_search_tool_deny_by_default.py b/tests/integration/authorization/test_search_tool_deny_by_default.py new file mode 100644 index 00000000000..f29940f68b2 --- /dev/null +++ b/tests/integration/authorization/test_search_tool_deny_by_default.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import json +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final, Literal, TypeAlias + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, Scenario, gateway_from_environment, object_value +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" +SEARCH_TOOL: Final = "integration-search" +OTHER_SEARCH_TOOL: Final = "integration-other-search" +FAILING_SEARCH_TOOL: Final = "integration-failing-search" +SEARCH_RESULT: Final = {"title": "Synthetic result", "url": "https://example.test/result", "content": "snippet"} +JsonObject: TypeAlias = dict[str, JsonValue] + + +def _json_array(*values: JsonValue) -> JsonValue: + return [*values] # mutable-ok: request payloads and YAML sequences require list values + + +def _grant(*search_tools: str) -> JsonObject: + permission: Final[JsonObject] = {"search_tools": _json_array(*search_tools)} + return permission + + +def _respond(request: Request) -> Reply: + if (request.method, request.target) == ("POST", "/failing/search"): + return Reply(status=500, body=b'{"error": "synthetic outage"}') + assert (request.method, request.target) == ("POST", "/tavily/search"), request + query: Final = json.loads(request.body)["query"] + return Reply(body=json.dumps({"query": query, "results": [SEARCH_RESULT]}).encode()) + + +def _config(directory: Path, wire: Wire) -> Path: + config: Final = object_value(yaml.safe_load(PROXY_CONFIG.read_text())) + tool_params: Final[JsonObject] = { + "search_provider": "tavily", + "api_key": "synthetic-tavily-key", + "api_base": f"{wire.url}/tavily", + } + failing_tool_params: Final[JsonObject] = {**tool_params, "api_base": f"{wire.url}/failing"} + strict: Final[JsonObject] = { + **config, + "general_settings": {**object_value(config["general_settings"]), "search_tool_deny_by_default": True}, + "router_settings": { + **object_value(config["router_settings"]), + "fallbacks": _json_array({FAILING_SEARCH_TOOL: _json_array(OTHER_SEARCH_TOOL)}), + }, + "search_tools": _json_array( + {"search_tool_name": SEARCH_TOOL, "litellm_params": tool_params}, + {"search_tool_name": OTHER_SEARCH_TOOL, "litellm_params": tool_params}, + {"search_tool_name": FAILING_SEARCH_TOOL, "litellm_params": failing_tool_params}, + ), + } + path: Final = directory / "proxy_search_tool_deny_by_default.yaml" + path.write_text(yaml.safe_dump(strict)) + return path + + +@pytest.fixture(scope="module") +def strict_search(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, Wire]]: + with gateway_from_environment() as upstream, wire_server(_respond) as wire: + directory: Final = tmp_path_factory.mktemp("search_tool_deny_by_default") + with owned_proxy(upstream, directory, {}, config=_config(directory, wire)) as gateway: + yield gateway, wire + + +def _search(gateway: Gateway, key: str, marker: str) -> httpx.Response: + return gateway.request("POST", f"/v1/search/{SEARCH_TOOL}", {"query": marker}, key=key) + + +def _searched(wire: Wire, marker: str) -> tuple[Request, ...]: + return tuple(request for request in wire.drain() if marker in request.body.decode()) + + +Caller: TypeAlias = Literal[ + "standalone_key_no_permission", + "standalone_key_empty_grant", + "standalone_key_other_tool", + "team_key_empty_key_grant", + "team_key_empty_team_grant", + "proxy_admin_key_no_grant", +] + + +def _denied_key(scenario: Scenario, caller: Caller) -> tuple[str, str]: + if caller == "standalone_key_no_permission": + return scenario.key(), "key_search_tool_access_denied" + if caller == "standalone_key_empty_grant": + return scenario.key(object_permission=_grant()), "key_search_tool_access_denied" + if caller == "standalone_key_other_tool": + return scenario.key(object_permission=_grant(OTHER_SEARCH_TOOL)), "key_search_tool_access_denied" + if caller == "team_key_empty_key_grant": + team: Final = scenario.team(object_permission=_grant(SEARCH_TOOL)) + return scenario.key(team_id=team, object_permission=_grant()), "key_search_tool_access_denied" + if caller == "team_key_empty_team_grant": + empty_team: Final = scenario.team(object_permission=_grant()) + return scenario.key(team_id=empty_team, object_permission=_grant(SEARCH_TOOL)), "team_search_tool_access_denied" + admin: Final = scenario.user(user_role="proxy_admin") + return scenario.key(user_id=admin), "key_search_tool_access_denied" + + +@pytest.mark.parametrize( + "caller", + [ + "standalone_key_no_permission", + "standalone_key_empty_grant", + "standalone_key_other_tool", + "team_key_empty_key_grant", + "team_key_empty_team_grant", + "proxy_admin_key_no_grant", + ], +) +def test_deny_by_default_rejects_ungranted_search_before_the_provider_is_called( + strict_search: tuple[Gateway, Wire], caller: Caller +) -> None: + gateway, wire = strict_search + with gateway.scenario() as scenario: + key, error_type = _denied_key(scenario, caller) + marker: Final = f"search deny {caller} {uuid.uuid4().hex}" + + response: Final = _search(gateway, key, marker) + + assert response.status_code == 403, response.text + assert response.json()["error"]["type"] == error_type, response.text + assert _searched(wire, marker) == () + + +@pytest.mark.parametrize("scope", ["standalone_key", "team_key", "master_key"]) +def test_deny_by_default_serves_explicitly_granted_search( + strict_search: tuple[Gateway, Wire], scope: Literal["standalone_key", "team_key", "master_key"] +) -> None: + gateway, wire = strict_search + with gateway.scenario() as scenario: + if scope == "standalone_key": + key = scenario.key(object_permission=_grant(SEARCH_TOOL)) + elif scope == "team_key": + team: Final = scenario.team(object_permission=_grant(SEARCH_TOOL)) + key = scenario.key(team_id=team, object_permission=_grant(SEARCH_TOOL)) + else: + key = gateway.key + marker: Final = f"search allow {scope} {uuid.uuid4().hex}" + + response: Final = _search(gateway, key, marker) + + assert response.status_code == 200, response.text + assert response.json()["results"][0]["url"] == SEARCH_RESULT["url"] + assert len(_searched(wire, marker)) == 1 + + +def test_search_tools_list_shows_only_the_tools_the_key_and_its_team_both_grant( + strict_search: tuple[Gateway, Wire], +) -> None: + gateway, _ = strict_search + with gateway.scenario() as scenario: + team: Final = scenario.team(object_permission=_grant(SEARCH_TOOL, OTHER_SEARCH_TOOL)) + member: Final = scenario.member(team) + key: Final = scenario.key(team_id=team, user_id=member, object_permission=_grant(SEARCH_TOOL)) + + response: Final = gateway.request("GET", "/search_tools/list", key=key) + + assert response.status_code == 200, response.text + assert [tool["search_tool_name"] for tool in response.json()["search_tools"]] == [SEARCH_TOOL] + + +def test_revoking_a_team_search_grant_takes_effect_on_the_next_request(strict_search: tuple[Gateway, Wire]) -> None: + gateway, wire = strict_search + with gateway.scenario() as scenario: + team: Final = scenario.team(object_permission=_grant(SEARCH_TOOL)) + key: Final = scenario.key(team_id=team, object_permission=_grant(SEARCH_TOOL)) + assert _search(gateway, key, f"search warm {uuid.uuid4().hex}").status_code == 200 + + gateway.post("/team/update", {"team_id": team, "object_permission": _grant()}) + marker: Final = f"search revoked {uuid.uuid4().hex}" + response: Final = _search(gateway, key, marker) + + assert response.status_code == 403, response.text + assert response.json()["error"]["type"] == "team_search_tool_access_denied", response.text + assert _searched(wire, marker) == () + + +def test_deny_by_default_authorizes_a_search_tool_named_by_the_model_field(strict_search: tuple[Gateway, Wire]) -> None: + gateway, wire = strict_search + with gateway.scenario() as scenario: + key: Final = scenario.key() + marker: Final = f"search model field {uuid.uuid4().hex}" + + response: Final = gateway.request("POST", "/v1/search", {"model": SEARCH_TOOL, "query": marker}, key=key) + + assert response.status_code == 403, response.text + assert response.json()["error"]["type"] == "key_search_tool_access_denied", response.text + assert _searched(wire, marker) == () + + +@pytest.mark.parametrize("fallback_granted", [False, True]) +def test_router_search_fallback_only_reaches_a_granted_tool( + strict_search: tuple[Gateway, Wire], fallback_granted: bool +) -> None: + gateway, wire = strict_search + with gateway.scenario() as scenario: + granted: Final = (FAILING_SEARCH_TOOL, OTHER_SEARCH_TOOL) if fallback_granted else (FAILING_SEARCH_TOOL,) + key: Final = scenario.key(object_permission=_grant(*granted)) + marker: Final = f"search fallback {fallback_granted} {uuid.uuid4().hex}" + + response: Final = gateway.request("POST", f"/v1/search/{FAILING_SEARCH_TOOL}", {"query": marker}, key=key) + + searched: Final = tuple(request.target for request in _searched(wire, marker)) + if fallback_granted: + assert response.status_code == 200, response.text + assert searched == ("/failing/search", "/tavily/search") + else: + assert response.status_code == 500, response.text + assert searched == ("/failing/search",) diff --git a/tests/unit/integrations/websearch_interception/test_websearch_interception_handler.py b/tests/unit/integrations/websearch_interception/test_websearch_interception_handler.py index b6a7c2ccb93..b7437b5fd8f 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_interception_handler.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_interception_handler.py @@ -13,7 +13,13 @@ from litellm.integrations.websearch_interception.handler import ( WebSearchInterceptionLogger, ) from litellm.llms.base_llm.search.transformation import SearchResponse -from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, ProxyException, UserAPIKeyAuth +from litellm.proxy._types import ( + LiteLLM_ObjectPermissionTable, + LiteLLM_TeamTable, + LitellmUserRoles, + ProxyException, + UserAPIKeyAuth, +) from litellm.types.utils import LlmProviders @@ -899,3 +905,184 @@ async def test_pre_request_hook_syncs_forced_tool_choice(): "type": "tool", "name": LITELLM_WEB_SEARCH_TOOL_NAME, } + + +def _single_search_tool_router(search_tool_name): + router = MagicMock() + router.search_tools = [ + { + "search_tool_name": search_tool_name, + "litellm_params": {"search_provider": "tavily", "api_key": "fake-ui-key"}, + } + ] + return router + + +def _virtual_key( + team_id: str | None = None, key_search_tools: list[str] | None = None, api_key: str = "sk-caller" +) -> UserAPIKeyAuth: + token = UserAPIKeyAuth( + api_key=api_key, + user_id="user-1", + team_id=team_id, + object_permission_id=None if key_search_tools is None else "op-key", + object_permission=( + None + if key_search_tools is None + else LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=key_search_tools) + ), + ) + token.via_virtual_key = True + return token + + +@pytest.mark.parametrize( + "general_settings, key_search_tools, team_search_tools, expect_search", + [ + ({}, None, [], True), + ({"search_tool_deny_by_default": True}, ["team-search"], [], False), + ({"search_tool_deny_by_default": True}, ["team-search"], None, False), + ({"search_tool_deny_by_default": True}, [], ["team-search"], False), + ({"search_tool_deny_by_default": True}, ["team-search"], ["team-search"], True), + ], +) +@pytest.mark.asyncio +async def test_execute_search_team_key_follows_search_tool_deny_by_default( + monkeypatch: pytest.MonkeyPatch, + general_settings: dict[str, bool], + key_search_tools: list[str] | None, + team_search_tools: list[str] | None, + expect_search: bool, +): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="team-search") + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + team_object = LiteLLM_TeamTable( + team_id="team-1", + object_permission_id=None if team_search_tools is None else "op-team", + object_permission=( + None + if team_search_tools is None + else LiteLLM_ObjectPermissionTable(object_permission_id="op-team", search_tools=team_search_tools) + ), + ) + + monkeypatch.setattr(proxy_server, "llm_router", _single_search_tool_router("team-search")) + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_team_object", AsyncMock(return_value=team_object)) + kwargs = {"metadata": {"user_api_key_auth": _virtual_key("team-1", key_search_tools)}} + + if expect_search: + await logger._execute_search("what is litellm", kwargs=kwargs) + mock_asearch.assert_awaited_once() + else: + with pytest.raises(ProxyException) as exc_info: + await logger._execute_search("what is litellm", kwargs=kwargs) + assert exc_info.value.code == "403" + mock_asearch.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_execute_search_key_without_grant_is_denied_under_search_tool_deny_by_default( + monkeypatch: pytest.MonkeyPatch, +): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="any-search") + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + + monkeypatch.setattr(proxy_server, "llm_router", _single_search_tool_router("any-search")) + monkeypatch.setattr(proxy_server, "general_settings", {"search_tool_deny_by_default": True}) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + with pytest.raises(ProxyException) as exc_info: + await logger._execute_search("what is litellm", kwargs={"metadata": {"user_api_key_auth": _virtual_key()}}) + + assert exc_info.value.code == "403" + mock_asearch.assert_not_awaited() + + +@pytest.mark.parametrize( + "general_settings, caller, expect_search", + [ + ({}, _virtual_key(), True), + ({"search_tool_deny_by_default": False}, _virtual_key(), True), + ({"search_tool_deny_by_default": True}, _virtual_key(key_search_tools=["any"]), False), + ( + {"search_tool_deny_by_default": True}, + UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + False, + ), + ({"search_tool_deny_by_default": True}, _virtual_key(api_key="litellm_proxy_master_key"), True), + ({"search_tool_deny_by_default": True}, None, True), + ], + ids=["omitted", "false", "virtual-key", "proxy-admin-not-exempt", "master-key-exempt", "sdk-without-proxy-auth"], +) +@pytest.mark.asyncio +async def test_execute_search_unregistered_fallback_follows_search_tool_deny_by_default( + monkeypatch: pytest.MonkeyPatch, + general_settings: dict[str, bool], + caller: UserAPIKeyAuth | None, + expect_search: bool, +): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + router = MagicMock() + router.search_tools = [] + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + kwargs = {"metadata": {} if caller is None else {"user_api_key_auth": caller}} + + if expect_search: + await logger._execute_search("what is litellm", kwargs=kwargs) + assert mock_asearch.await_args.kwargs["search_provider"] == "perplexity" + else: + with pytest.raises(ProxyException) as exc_info: + await logger._execute_search("what is litellm", kwargs=kwargs) + assert exc_info.value.code == "403" + mock_asearch.assert_not_awaited() + + +@pytest.mark.parametrize( + "general_settings, expect_search", + [({}, True), ({"search_tool_deny_by_default": True}, False)], +) +@pytest.mark.asyncio +async def test_execute_search_registered_tool_without_provider_follows_unregistered_fallback_policy( + monkeypatch: pytest.MonkeyPatch, + general_settings: dict[str, bool], + expect_search: bool, +): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="no-provider") + router = MagicMock() + router.search_tools = [{"search_tool_name": "no-provider", "litellm_params": {}}] + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + kwargs = {"metadata": {"user_api_key_auth": _virtual_key(key_search_tools=["no-provider"])}} + + if expect_search: + await logger._execute_search("what is litellm", kwargs=kwargs) + assert mock_asearch.await_args.kwargs["search_provider"] == "perplexity" + else: + with pytest.raises(ProxyException) as exc_info: + await logger._execute_search("what is litellm", kwargs=kwargs) + assert exc_info.value.code == "403" + mock_asearch.assert_not_awaited() diff --git a/tests/unit/proxy/auth/test_fallback_model_access.py b/tests/unit/proxy/auth/test_fallback_model_access.py index 4cbf4474596..24fcd4671c8 100644 --- a/tests/unit/proxy/auth/test_fallback_model_access.py +++ b/tests/unit/proxy/auth/test_fallback_model_access.py @@ -1,12 +1,13 @@ import pytest from litellm import Router -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth from litellm.proxy.auth.fallback_model_access import ( RouterFallbackAccessCheck, is_model_authorized_for_token, router_fallback_access_check, ) +from litellm.search import asearch def _router() -> Router: @@ -105,3 +106,69 @@ async def test_proxy_check_reads_enforce_fallback_model_access_from_general_sett ) is expected ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("check", [ENFORCED, NOT_ENFORCED], ids=["enforced", "not-enforced"]) +async def test_search_tool_fallback_target_follows_the_key_search_tool_grant( + monkeypatch: pytest.MonkeyPatch, check: RouterFallbackAccessCheck +): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + router = Router( + model_list=[], + search_tools=[ + {"search_tool_name": name, "litellm_params": {"search_provider": "tavily", "api_key": "k"}} + for name in ("search-a", "search-b") + ], + ) + request_kwargs = { + "original_generic_function": asearch, + "litellm_metadata": { + "user_api_key_auth": UserAPIKeyAuth( + api_key="hashed", + object_permission_id="op-key", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="op-key", search_tools=["search-a"] + ), + ) + }, + } + + assert await check(model="search-a", request_kwargs=request_kwargs, llm_router=router) + assert not await check(model="search-b", request_kwargs=request_kwargs, llm_router=router) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_kwargs, expected", + [ + ({"original_generic_function": asearch}, True), + ({}, False), + ], + ids=["search-request", "completion-request"], +) +async def test_fallback_named_like_both_a_model_and_a_search_tool_follows_the_request_kind( + request_kwargs: dict, expected: bool +): + router = Router( + model_list=[ + {"model_name": "shared-name", "litellm_params": {"model": "openai/secret", "api_key": "k"}}, + ], + search_tools=[ + {"search_tool_name": "shared-name", "litellm_params": {"search_provider": "tavily", "api_key": "k"}}, + ], + ) + key = UserAPIKeyAuth( + api_key="hashed", + models=["open-model"], + object_permission_id="op-key", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=["shared-name"]), + ) + + allowed = await ENFORCED( + model="shared-name", + request_kwargs={**request_kwargs, "metadata": {"user_api_key_auth": key}}, + llm_router=router, + ) + + assert allowed is expected diff --git a/tests/unit/proxy/auth/test_search_tool_deny_by_default.py b/tests/unit/proxy/auth/test_search_tool_deny_by_default.py new file mode 100644 index 00000000000..cd285e0386f --- /dev/null +++ b/tests/unit/proxy/auth/test_search_tool_deny_by_default.py @@ -0,0 +1,270 @@ +from collections.abc import Mapping +from typing import Final, TypedDict +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS +from litellm.proxy import proxy_server +from litellm.proxy._types import ( + UI_TEAM_ID, + LiteLLM_ObjectPermissionTable, + LiteLLM_TeamTable, + LiteLLM_UserTable, + LitellmUserRoles, + ProxyErrorTypes, + ProxyException, + UserAPIKeyAuth, +) +from litellm.proxy.auth.auth_checks import ( + TeamObjectLoader, + can_caller_call_search_tool, + check_unregistered_search_fallback, +) +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, object_permission_cache_key + +_DENY_ON: Final = {"search_tool_deny_by_default": True} +_KEY: Final = ProxyErrorTypes.key_search_tool_access_denied +_TEAM: Final = ProxyErrorTypes.team_search_tool_access_denied +_USER: Final = ProxyErrorTypes.user_search_tool_access_denied +_LEGACY: Final = ProxyErrorTypes.key_model_access_denied +_TEAM_LOAD_FAILS: Final = "team-load-fails" +_NO_ROW: Final = "no-row" + + +class CallerFields(TypedDict, total=False): + virtual_key: bool + key_tools: list[str] | None | str + team_id: str | None + user_id: str | None + user_role: LitellmUserRoles | None + api_key: str + + +def _grant(search_tools: list[str] | None, object_permission_id: str = "op") -> LiteLLM_ObjectPermissionTable: + return LiteLLM_ObjectPermissionTable(object_permission_id=object_permission_id, search_tools=search_tools) + + +def _caller( + virtual_key: bool = True, + key_tools: list[str] | None | str = _NO_ROW, + team_id: str | None = None, + user_id: str | None = "user-1", + user_role: LitellmUserRoles | None = LitellmUserRoles.INTERNAL_USER, + api_key: str = "sk-caller", +) -> UserAPIKeyAuth: + token: Final = UserAPIKeyAuth( + api_key=api_key, + user_id=user_id, + user_role=user_role, + team_id=team_id, + object_permission_id=None if key_tools == _NO_ROW else "op-key", + object_permission=None if key_tools == _NO_ROW else _grant(key_tools, "op-key"), + ) + token.via_virtual_key = virtual_key + return token + + +def _team_loader(team_tools: list[str] | None | str) -> TeamObjectLoader: + async def load() -> LiteLLM_TeamTable | None: + if team_tools == _TEAM_LOAD_FAILS: + raise ProxyException(message="team lookup failed", type="auth_error", param="team_id", code=404) + if team_tools == _NO_ROW: + return LiteLLM_TeamTable(team_id="team-1") + return LiteLLM_TeamTable( + team_id="team-1", object_permission_id="op-team", object_permission=_grant(team_tools, "op-team") + ) + + return load + + +async def _no_team() -> None: + return None + + +@pytest.fixture +def cache(monkeypatch: pytest.MonkeyPatch) -> UserApiKeyCache: + user_api_key_cache: Final = UserApiKeyCache() + monkeypatch.setattr(proxy_server, "user_api_key_cache", user_api_key_cache) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + return user_api_key_cache + + +def _cache_user(cache: UserApiKeyCache, user_tools: list[str] | None) -> None: + cache.set_cache(key="user-1", value=LiteLLM_UserTable(user_id="user-1", object_permission_id="op-user")) + cache.set_cache(key=object_permission_cache_key("op-user"), value=_grant(user_tools, "op-user")) + + +async def _denied_by( + general_settings: Mapping[str, object], caller: UserAPIKeyAuth, load_team: TeamObjectLoader = _no_team +) -> ProxyErrorTypes | None: + try: + await can_caller_call_search_tool("search-a", caller, general_settings, load_team) + except ProxyException as e: + denial = e + else: + return None + assert (denial.code, denial.param) == ("403", "search_tool_name") + return ProxyErrorTypes(denial.type) + + +@pytest.mark.parametrize( + "general_settings", + [{}, {"search_tool_deny_by_default": False}], + ids=["omitted", "false"], +) +@pytest.mark.parametrize( + "caller, team_tools, expected", + [ + pytest.param(CallerFields(), None, None, id="no grants anywhere"), + pytest.param(CallerFields(key_tools=[]), None, None, id="empty key list"), + pytest.param(CallerFields(team_id="team-1"), [], None, id="empty team list"), + pytest.param(CallerFields(key_tools=["search-b"]), None, _LEGACY, id="key allowlist excludes"), + pytest.param(CallerFields(team_id="team-1"), ["search-b"], _LEGACY, id="team allowlist excludes"), + pytest.param(CallerFields(key_tools=["search-a"], team_id="team-1"), ["search-a"], None, id="both allow"), + ], +) +@pytest.mark.asyncio +async def test_search_tool_access_is_unchanged_without_search_tool_deny_by_default( + cache: UserApiKeyCache, + general_settings: dict[str, bool], + caller: CallerFields, + team_tools: list[str] | None, + expected: ProxyErrorTypes | None, +): + _cache_user(cache, ["search-b"]) + load_team: Final = _no_team if team_tools is None else _team_loader(team_tools) + assert await _denied_by(general_settings, _caller(**caller), load_team) == expected + + +@pytest.mark.parametrize( + "caller, team_tools, user_tools, expected", + [ + pytest.param(CallerFields(), None, ["search-a"], _KEY, id="standalone key, no permission row"), + pytest.param(CallerFields(key_tools=None), None, None, _KEY, id="standalone key, search_tools null"), + pytest.param(CallerFields(key_tools=[]), None, None, _KEY, id="standalone key, empty grant"), + pytest.param(CallerFields(key_tools=["search-b"]), None, None, _KEY, id="standalone key, grants another tool"), + pytest.param(CallerFields(key_tools=["search-a"]), None, [], None, id="standalone key, key grant is enough"), + pytest.param(CallerFields(key_tools=[]), None, ["search-a"], _KEY, id="standalone key, user cannot stand in"), + pytest.param( + CallerFields(key_tools=["search-a"], team_id="team-1"), ["search-a"], None, None, id="team key, both" + ), + pytest.param(CallerFields(key_tools=[], team_id="team-1"), ["search-a"], None, _KEY, id="team key, empty key"), + pytest.param( + CallerFields(key_tools=["search-a"], team_id="team-1"), [], None, _TEAM, id="team key, empty team" + ), + pytest.param( + CallerFields(key_tools=["search-a"], team_id="team-1"), None, None, _TEAM, id="team key, team null" + ), + pytest.param( + CallerFields(key_tools=["search-a"], team_id="team-1"), _NO_ROW, None, _TEAM, id="team key, team has no row" + ), + pytest.param( + CallerFields(key_tools=["search-a"], team_id="team-1"), + _TEAM_LOAD_FAILS, + None, + _TEAM, + id="team key, team fails to load", + ), + pytest.param( + CallerFields(key_tools=["search-a"], team_id="team-1"), + ["search-b"], + ["search-a"], + _TEAM, + id="team key, user cannot stand in for team", + ), + pytest.param( + CallerFields(virtual_key=False, team_id="team-1"), ["search-a"], [], None, id="keyless team member" + ), + pytest.param( + CallerFields(virtual_key=False, team_id="team-1"), [], ["search-a"], _TEAM, id="keyless, empty team" + ), + pytest.param(CallerFields(virtual_key=False), None, ["search-a"], None, id="keyless user, user grants"), + pytest.param(CallerFields(virtual_key=False), None, None, _USER, id="keyless user, search_tools null"), + pytest.param(CallerFields(virtual_key=False), None, [], _USER, id="keyless user, empty grant"), + pytest.param( + CallerFields(virtual_key=False, user_id="user-2"), None, None, _USER, id="keyless user fails to load" + ), + pytest.param(CallerFields(virtual_key=False, user_id=None), None, None, _USER, id="keyless caller, no user"), + pytest.param( + CallerFields(user_role=LitellmUserRoles.PROXY_ADMIN), + None, + None, + _KEY, + id="proxy admin virtual key not exempt", + ), + pytest.param(CallerFields(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS), None, None, None, id="master key exempt"), + pytest.param( + CallerFields(team_id=UI_TEAM_ID, virtual_key=False), None, None, None, id="dashboard session exempt" + ), + ], +) +@pytest.mark.asyncio +async def test_search_tool_deny_by_default_requires_every_owning_identity_to_grant( + cache: UserApiKeyCache, + caller: CallerFields, + team_tools: list[str] | str | None, + user_tools: list[str] | None, + expected: ProxyErrorTypes | None, +): + if user_tools is not None: + _cache_user(cache, user_tools) + elif caller.get("user_id", "user-1") == "user-1": + _cache_user(cache, None) + load_team: Final = _no_team if team_tools is None and "team_id" not in caller else _team_loader(team_tools) + assert await _denied_by(_DENY_ON, _caller(**caller), load_team) == expected + + +@pytest.mark.parametrize("value", ["true", "enabled", 1], ids=["string-true", "string", "int"]) +@pytest.mark.asyncio +async def test_non_boolean_search_tool_deny_by_default_enables_the_policy(cache: UserApiKeyCache, value: object): + assert await _denied_by({"search_tool_deny_by_default": value}, _caller(key_tools=[])) == _KEY + + +@pytest.mark.asyncio +async def test_search_tool_deny_by_default_denies_when_no_database_is_connected( + cache: UserApiKeyCache, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(proxy_server, "prisma_client", None) + assert await _denied_by(_DENY_ON, _caller(key_tools=["search-a"])) == _KEY + + +@pytest.mark.asyncio +async def test_search_tool_deny_by_default_reads_the_user_grant_on_every_call(cache: UserApiKeyCache): + caller: Final = _caller(virtual_key=False) + _cache_user(cache, ["search-a"]) + assert await _denied_by(_DENY_ON, caller) is None + cache.set_cache(key=object_permission_cache_key("op-user"), value=_grant([], "op-user")) + assert await _denied_by(_DENY_ON, caller) == _USER + + +@pytest.mark.asyncio +async def test_search_tool_deny_by_default_does_not_load_the_user_when_off( + cache: UserApiKeyCache, monkeypatch: pytest.MonkeyPatch +): + get_user_object: Final = AsyncMock() + monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_user_object", get_user_object) + assert await _denied_by({}, _caller(virtual_key=False)) is None + get_user_object.assert_not_awaited() + + +@pytest.mark.parametrize( + "general_settings, caller, expected", + [ + pytest.param({}, _caller(), None, id="flag off"), + pytest.param(_DENY_ON, _caller(), _KEY, id="virtual key"), + pytest.param(_DENY_ON, _caller(virtual_key=False, team_id="team-1"), _TEAM, id="keyless team member"), + pytest.param(_DENY_ON, _caller(virtual_key=False), _USER, id="keyless user"), + pytest.param(_DENY_ON, _caller(user_role=LitellmUserRoles.PROXY_ADMIN), _KEY, id="proxy admin key"), + pytest.param(_DENY_ON, _caller(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS), None, id="master key"), + ], +) +def test_unregistered_search_fallback_follows_search_tool_deny_by_default( + general_settings: Mapping[str, object], caller: UserAPIKeyAuth, expected: ProxyErrorTypes | None +): + if expected is None: + assert check_unregistered_search_fallback(caller, general_settings) is True + return + with pytest.raises(ProxyException) as exc_info: + check_unregistered_search_fallback(caller, general_settings) + assert (exc_info.value.type, exc_info.value.code) == (expected, "403") diff --git a/tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py b/tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py index 582f6e82f11..49bc2d0f64e 100644 --- a/tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py +++ b/tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py @@ -1,14 +1,16 @@ import contextlib import json -from types import SimpleNamespace +from collections.abc import Callable from datetime import datetime +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException from fastapi.testclient import TestClient - +# Import proxy_server module first to ensure it's initialized +import litellm.proxy.proxy_server as ps from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, @@ -16,9 +18,6 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) -# Import proxy_server module first to ensure it's initialized -import litellm.proxy.proxy_server as ps - # Now we can safely import app from litellm.proxy.proxy_server import app from litellm.types.search import SearchToolInfoResponse @@ -116,9 +115,7 @@ async def test_list_search_tools_config_only(monkeypatch): with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): # Mock proxy_config mock_proxy_config = MagicMock() - mock_proxy_config.get_config = AsyncMock( - return_value={"search_tools": config_tools} - ) + mock_proxy_config.get_config = AsyncMock(return_value={"search_tools": config_tools}) mock_proxy_config.parse_search_tools = MagicMock(return_value=config_tools) with patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config): # Mock auth @@ -192,9 +189,7 @@ async def test_list_search_tools_filters_duplicate_config_tools(monkeypatch): with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): # Mock proxy_config mock_proxy_config = MagicMock() - mock_proxy_config.get_config = AsyncMock( - return_value={"search_tools": config_tools} - ) + mock_proxy_config.get_config = AsyncMock(return_value={"search_tools": config_tools}) mock_proxy_config.parse_search_tools = MagicMock(return_value=config_tools) with patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config): # Mock auth @@ -215,11 +210,7 @@ async def test_list_search_tools_filters_duplicate_config_tools(monkeypatch): # Verify DB tool is present db_tool = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "existing-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "existing-tool"), None, ) assert db_tool is not None @@ -232,11 +223,7 @@ async def test_list_search_tools_filters_duplicate_config_tools(monkeypatch): # Verify unique config tool is present config_tool = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "unique-config-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "unique-config-tool"), None, ) assert config_tool is not None @@ -247,8 +234,7 @@ async def test_list_search_tools_filters_duplicate_config_tools(monkeypatch): ( t for t in data["search_tools"] - if t["search_tool_name"] == "existing-tool" - and t["is_from_config"] is True + if t["search_tool_name"] == "existing-tool" and t["is_from_config"] is True ), None, ) @@ -323,11 +309,7 @@ async def test_list_search_tools_datetime_conversion(monkeypatch): # Test datetime conversion for tool 1 tool1 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "datetime-test-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "datetime-test-tool"), None, ) assert tool1 is not None @@ -341,11 +323,7 @@ async def test_list_search_tools_datetime_conversion(monkeypatch): # Test None handling for tool 2 tool2 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "null-datetime-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "null-datetime-tool"), None, ) assert tool2 is not None @@ -357,11 +335,7 @@ async def test_list_search_tools_datetime_conversion(monkeypatch): # Test string passthrough for tool 3 tool3 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "string-datetime-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "string-datetime-tool"), None, ) assert tool3 is not None @@ -401,9 +375,7 @@ async def test_list_search_tools_config_error_handling(monkeypatch): with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): # Mock proxy_config to raise an error mock_proxy_config = MagicMock() - mock_proxy_config.get_config = AsyncMock( - side_effect=Exception("Config error") - ) + mock_proxy_config.get_config = AsyncMock(side_effect=Exception("Config error")) with patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config): # Mock auth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -422,13 +394,8 @@ async def test_list_search_tools_config_error_handling(monkeypatch): assert len(data["search_tools"]) == 1 assert data["search_tools"][0]["search_tool_name"] == "db-tool-1" # Verify masking of sensitive values - assert ( - data["search_tools"][0]["litellm_params"]["api_key"] - != "sk-test" - ) - assert ( - "****" in data["search_tools"][0]["litellm_params"]["api_key"] - ) + assert data["search_tools"][0]["litellm_params"]["api_key"] != "sk-test" + assert "****" in data["search_tools"][0]["litellm_params"]["api_key"] finally: app.dependency_overrides.pop(user_api_key_auth, None) @@ -543,31 +510,18 @@ async def test_list_search_tools_db_masking_sensitive_values(monkeypatch): # Test tool 1: api_key should be masked tool1 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "perplexity-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "perplexity-tool"), None, ) assert tool1 is not None - assert ( - tool1["litellm_params"]["api_key"] != "pplx-sk-9876567890abcdef" - ) + assert tool1["litellm_params"]["api_key"] != "pplx-sk-9876567890abcdef" assert "****" in tool1["litellm_params"]["api_key"] assert tool1["litellm_params"]["search_provider"] == "perplexity" - assert ( - tool1["litellm_params"]["api_base"] - == "https://api.perplexity.ai" - ) + assert tool1["litellm_params"]["api_base"] == "https://api.perplexity.ai" # Test tool 2: api_key should be masked tool2 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "tavily-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "tavily-tool"), None, ) assert tool2 is not None @@ -577,29 +531,18 @@ async def test_list_search_tools_db_masking_sensitive_values(monkeypatch): # Test tool 3: access_token and secret_key should be masked tool3 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "tool-with-token" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "tool-with-token"), None, ) assert tool3 is not None - assert ( - tool3["litellm_params"]["access_token"] - != "token-abcdefghijklmnop" - ) + assert tool3["litellm_params"]["access_token"] != "token-abcdefghijklmnop" assert "****" in tool3["litellm_params"]["access_token"] assert tool3["litellm_params"]["secret_key"] != "secret-xyz123" assert "****" in tool3["litellm_params"]["secret_key"] # Test tool 4: non-sensitive fields should remain unmasked tool4 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "tool-with-non-sensitive" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "tool-with-non-sensitive"), None, ) assert tool4 is not None @@ -615,6 +558,7 @@ async def test_get_all_search_tools_from_db_retries_on_transport_error(): """`SearchToolRegistry.get_all_search_tools_from_db` self-heals across one ClientNotConnectedError via call_with_db_reconnect_retry.""" import prisma + from litellm.proxy.search_endpoints.search_tool_registry import ( SearchToolRegistry, ) @@ -628,25 +572,18 @@ async def test_get_all_search_tools_from_db_retries_on_transport_error(): return [] mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_searchtoolstable.find_many = AsyncMock( - side_effect=_flaky_find_many - ) + mock_prisma_client.db.litellm_searchtoolstable.find_many = AsyncMock(side_effect=_flaky_find_many) mock_prisma_client.attempt_db_reconnect = AsyncMock(return_value=True) mock_prisma_client._db_auth_reconnect_timeout_seconds = 2.0 mock_prisma_client._db_auth_reconnect_lock_timeout_seconds = 0.1 - result = await SearchToolRegistry.get_all_search_tools_from_db( - prisma_client=mock_prisma_client - ) + result = await SearchToolRegistry.get_all_search_tools_from_db(prisma_client=mock_prisma_client) assert result == [] assert len(invocations) == 2 mock_prisma_client.attempt_db_reconnect.assert_awaited_once() reconnect_kwargs = mock_prisma_client.attempt_db_reconnect.await_args.kwargs - assert ( - reconnect_kwargs["reason"] - == "get_all_search_tools_from_db_lookup_failure" - ) + assert reconnect_kwargs["reason"] == "get_all_search_tools_from_db_lookup_failure" @contextlib.contextmanager @@ -749,9 +686,7 @@ async def test_list_search_tools_scoped_to_key_object_permission(): @pytest.mark.asyncio async def test_list_search_tools_unrestricted_internal_user_sees_all(): """An internal user with no search_tools allowlist is unrestricted and sees every tool.""" - unrestricted_user = UserAPIKeyAuth( - user_role=LitellmUserRoles.INTERNAL_USER, user_id="internal_user" - ) + unrestricted_user = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="internal_user") with ( _mock_search_tool_backend(_scoping_db_tools()), @@ -791,9 +726,7 @@ async def test_list_search_tools_scoped_to_team_object_permission(): response = TestClient(app).get("/search_tools/list") assert response.status_code == 200 - assert [t["search_tool_name"] for t in response.json()["search_tools"]] == [ - "db-tool-2" - ] + assert [t["search_tool_name"] for t in response.json()["search_tools"]] == ["db-tool-2"] @pytest.mark.asyncio @@ -1061,9 +994,15 @@ def _live_router_and_db(db_rows: list): fake_router.search_tools = list(db_rows) with contextlib.ExitStack() as stack: - stack.enter_context(patch("litellm.proxy.proxy_server.prisma_client", MagicMock())) # test-quality-ok: proxy globals are the only seam; see the module note above - stack.enter_context(patch("litellm.proxy.proxy_server.proxy_config", proxy_config)) # test-quality-ok: proxy globals are the only seam; see the module note above - stack.enter_context(patch("litellm.proxy.proxy_server.llm_router", fake_router)) # test-quality-ok: proxy globals are the only seam; see the module note above + stack.enter_context( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()) + ) # test-quality-ok: proxy globals are the only seam; see the module note above + stack.enter_context( + patch("litellm.proxy.proxy_server.proxy_config", proxy_config) + ) # test-quality-ok: proxy globals are the only seam; see the module note above + stack.enter_context( + patch("litellm.proxy.proxy_server.llm_router", fake_router) + ) # test-quality-ok: proxy globals are the only seam; see the module note above stack.enter_context( patch( # test-quality-ok: proxy globals are the only seam; see the module note above "litellm.proxy.search_endpoints.search_tool_management.SEARCH_TOOL_REGISTRY", @@ -1152,6 +1091,171 @@ async def test_create_search_tool_survives_a_failing_router_refresh(): assert response.json()["search_tool_name"] == "tavily-search" +def _search_caller( + virtual_key: bool, + key_search_tools: list[str] | None = None, + team_id: str | None = None, + user_role: LitellmUserRoles = LitellmUserRoles.INTERNAL_USER, + api_key: str = "sk-caller", +) -> UserAPIKeyAuth: + caller = UserAPIKeyAuth( + api_key=api_key, + user_role=user_role, + user_id="internal_user", + team_id=team_id, + object_permission_id=None if key_search_tools is None else "op-key", + object_permission=( + None + if key_search_tools is None + else LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=key_search_tools) + ), + ) + caller.via_virtual_key = virtual_key + return caller + + +@pytest.fixture +def search_permission_cache(monkeypatch: pytest.MonkeyPatch) -> Callable[[list[str] | None], None]: + from litellm.proxy._types import LiteLLM_UserTable + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, object_permission_cache_key + + cache = UserApiKeyCache() + monkeypatch.setattr(ps, "user_api_key_cache", cache) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + + def grant_user(search_tools: list[str] | None) -> None: + cache.set_cache( + key="internal_user", value=LiteLLM_UserTable(user_id="internal_user", object_permission_id="op-user") + ) + cache.set_cache( + key=object_permission_cache_key("op-user"), + value=LiteLLM_ObjectPermissionTable(object_permission_id="op-user", search_tools=search_tools), + ) + + return grant_user + + +@pytest.mark.parametrize( + "general_settings, user_search_tools, expected", + [ + ({}, None, ["db-tool-1", "db-tool-2"]), + ({"search_tool_deny_by_default": False}, [], ["db-tool-1", "db-tool-2"]), + ({"search_tool_deny_by_default": True}, None, []), + ({"search_tool_deny_by_default": True}, [], []), + ({"search_tool_deny_by_default": True}, ["db-tool-2"], ["db-tool-2"]), + ], +) +@pytest.mark.asyncio +async def test_filter_visible_search_tools_keyless_user_follows_search_tool_deny_by_default( + search_permission_cache: Callable[[list[str] | None], None], + general_settings: dict[str, bool], + user_search_tools: list[str] | None, + expected: list[str], +): + from litellm.proxy.search_endpoints.search_tool_management import _filter_visible_search_tools + + search_permission_cache(user_search_tools) + team_lookup = AsyncMock() + + visible = await _filter_visible_search_tools( + _search_tool_responses("db-tool-1", "db-tool-2"), + _search_caller(virtual_key=False), + team_lookup, + general_settings, + ) + + assert [t["search_tool_name"] for t in visible] == expected + team_lookup.assert_not_awaited() + + +@pytest.mark.parametrize( + "key_search_tools, expected", + [(None, []), ([], []), (["db-tool-1"], ["db-tool-1"])], +) +@pytest.mark.asyncio +async def test_filter_visible_search_tools_standalone_key_needs_its_own_grant( + search_permission_cache: Callable[[list[str] | None], None], key_search_tools: list[str] | None, expected: list[str] +): + from litellm.proxy.search_endpoints.search_tool_management import _filter_visible_search_tools + + search_permission_cache(["db-tool-1", "db-tool-2"]) + + visible = await _filter_visible_search_tools( + _search_tool_responses("db-tool-1", "db-tool-2"), + _search_caller(virtual_key=True, key_search_tools=key_search_tools), + AsyncMock(), + {"search_tool_deny_by_default": True}, + ) + + assert [t["search_tool_name"] for t in visible] == expected + + +@pytest.mark.parametrize( + "key_search_tools, team_search_tools, expected", + [ + (["db-tool-1"], [], []), + ([], ["db-tool-1"], []), + (["db-tool-1", "db-tool-2"], ["db-tool-1"], ["db-tool-1"]), + ], +) +@pytest.mark.asyncio +async def test_filter_visible_search_tools_team_key_needs_key_and_team_grants( + search_permission_cache: Callable[[list[str] | None], None], + key_search_tools: list[str], + team_search_tools: list[str], + expected: list[str], +): + from litellm.proxy.search_endpoints.search_tool_management import _filter_visible_search_tools + + team_lookup = AsyncMock( + return_value=LiteLLM_TeamTable( + team_id="team-1", + object_permission_id="op-team", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="op-team", search_tools=team_search_tools + ), + ) + ) + + visible = await _filter_visible_search_tools( + _search_tool_responses("db-tool-1", "db-tool-2"), + _search_caller(virtual_key=True, key_search_tools=key_search_tools, team_id="team-1"), + team_lookup, + {"search_tool_deny_by_default": True}, + ) + + assert [t["search_tool_name"] for t in visible] == expected + + +@pytest.mark.parametrize( + "caller, expected", + [ + ( + _search_caller( + virtual_key=False, user_role=LitellmUserRoles.PROXY_ADMIN, api_key="litellm_proxy_master_key" + ), + ["db-tool-1", "db-tool-2"], + ), + (_search_caller(virtual_key=True, user_role=LitellmUserRoles.PROXY_ADMIN), []), + ], + ids=["master-key-sees-every-tool", "admin-virtual-key-is-filtered"], +) +@pytest.mark.asyncio +async def test_filter_visible_search_tools_admin_under_search_tool_deny_by_default( + search_permission_cache: Callable[[list[str] | None], None], caller: UserAPIKeyAuth, expected: list[str] +): + from litellm.proxy.search_endpoints.search_tool_management import _filter_visible_search_tools + + visible = await _filter_visible_search_tools( + _search_tool_responses("db-tool-1", "db-tool-2"), + caller, + AsyncMock(), + {"search_tool_deny_by_default": True}, + ) + + assert [t["search_tool_name"] for t in visible] == expected + + class _StoredSearchToolRow(SimpleNamespace): def __iter__(self): return iter(self.__dict__.items()) diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index b37576dab80..0b58e5d141f 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -2330,38 +2330,36 @@ async def test_load_config_logs_disabled_budget_reservation_once(tmp_path, monke @pytest.mark.asyncio +@pytest.mark.parametrize("setting", ["vector_store_deny_by_default", "search_tool_deny_by_default"]) @pytest.mark.parametrize(("yaml_value", "expected"), [("true", True), ("false", False)]) -async def test_load_config_yaml_vector_store_deny_by_default_is_boolean( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, yaml_value: str, expected: bool +async def test_load_config_yaml_deny_by_default_is_boolean( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, setting: str, yaml_value: str, expected: bool ): - config_file: Final = tmp_path / "vector_store.yaml" - config_file.write_text( - f"model_list: []\nlitellm_settings: {{}}\ngeneral_settings:\n vector_store_deny_by_default: {yaml_value}\n" - ) + config_file: Final = tmp_path / "deny_by_default.yaml" + config_file.write_text(f"model_list: []\nlitellm_settings: {{}}\ngeneral_settings:\n {setting}: {yaml_value}\n") monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) _, _, general_settings = await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) - assert general_settings["vector_store_deny_by_default"] is expected - assert ConfigGeneralSettings.model_validate(dict(general_settings)).vector_store_deny_by_default is expected + assert general_settings[setting] is expected + assert getattr(ConfigGeneralSettings.model_validate(dict(general_settings)), setting) is expected @pytest.mark.asyncio +@pytest.mark.parametrize("setting", ["vector_store_deny_by_default", "search_tool_deny_by_default"]) @pytest.mark.parametrize("yaml_value", ["", "enabled"], ids=["null", "string"]) -async def test_load_config_rejects_non_boolean_vector_store_deny_by_default( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, yaml_value: str +async def test_load_config_rejects_non_boolean_deny_by_default( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, setting: str, yaml_value: str ): - config_file: Final = tmp_path / "vector_store.yaml" - config_file.write_text( - f"model_list: []\nlitellm_settings: {{}}\ngeneral_settings:\n vector_store_deny_by_default: {yaml_value}\n" - ) + config_file: Final = tmp_path / "deny_by_default.yaml" + config_file.write_text(f"model_list: []\nlitellm_settings: {{}}\ngeneral_settings:\n {setting}: {yaml_value}\n") monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) - with pytest.raises(ValidationError, match="vector_store_deny_by_default"): + with pytest.raises(ValidationError, match=setting): await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) @@ -3652,9 +3650,7 @@ def test_ProxyConfig__decrypt_and_set_db_env_variables_cannot_enable_mcp_stdio(m assert os.environ.get("LITELLM_ENABLE_MCP_STDIO") is None -def test_ProxyConfig__decrypt_and_set_db_env_variables_warns_once_about_the_ignored_mcp_stdio_flag( - monkeypatch, caplog -): +def test_ProxyConfig__decrypt_and_set_db_env_variables_warns_once_about_the_ignored_mcp_stdio_flag(monkeypatch, caplog): monkeypatch.setattr( "litellm.proxy.proxy_server.decrypt_value_helper", lambda value, key, return_original_value=False: value, diff --git a/tests/unit/proxy/search_endpoints/test_endpoints.py b/tests/unit/proxy/search_endpoints/test_endpoints.py index bd6460e3dfb..e8212b2a8b1 100644 --- a/tests/unit/proxy/search_endpoints/test_endpoints.py +++ b/tests/unit/proxy/search_endpoints/test_endpoints.py @@ -1,12 +1,142 @@ +from collections.abc import Iterator +from typing import Final from unittest.mock import AsyncMock, MagicMock import orjson import pytest +import respx +from fastapi import FastAPI +from fastapi.testclient import TestClient +import litellm +from litellm import Router from litellm.proxy import proxy_server -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ( + LiteLLM_ObjectPermissionTable, + LiteLLM_TeamTableCachedObj, + LitellmUserRoles, + ProxyException, + UserAPIKeyAuth, +) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.route_llm_request import ProxyMissingRequiredParamError -from litellm.proxy.search_endpoints.endpoints import search +from litellm.proxy.search_endpoints.endpoints import router, search + +TAVILY_SEARCH_URL: Final = "https://api.tavily.com/search" +TAVILY_RESULT: Final = {"title": "LiteLLM", "url": "https://docs.litellm.ai", "content": "LLM gateway"} + + +def _search_router() -> Router: + return Router( + model_list=[], + search_tools=[ + {"search_tool_name": "search-a", "litellm_params": {"search_provider": "tavily", "api_key": "fake"}}, + ], + num_retries=0, + ) + + +def _client(caller: UserAPIKeyAuth) -> TestClient: + app: Final = FastAPI() + app.include_router(router) + app.add_exception_handler(ProxyException, proxy_server.openai_exception_handler) + app.dependency_overrides[user_api_key_auth] = lambda: caller + return TestClient(app, raise_server_exceptions=False) + + +def _team_key(key_search_tools: list[str]) -> UserAPIKeyAuth: + caller: Final = UserAPIKeyAuth( + api_key="sk-team-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="user-1", + team_id="team-1", + object_permission_id="op-key", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=key_search_tools), + ) + caller.via_virtual_key = True + return caller + + +@pytest.fixture +def tavily(monkeypatch: pytest.MonkeyPatch) -> Iterator[respx.Route]: + monkeypatch.setattr( # test-quality-ok: respx needs HTTPX enabled to fake the provider HTTP boundary. + litellm, + "disable_aiohttp_transport", + True, + ) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=False) as mock: + yield mock.post(TAVILY_SEARCH_URL).respond(200, json={"results": [TAVILY_RESULT]}) + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.fixture +def cache(monkeypatch: pytest.MonkeyPatch) -> UserApiKeyCache: + user_api_key_cache: Final = UserApiKeyCache() + monkeypatch.setattr(proxy_server, "user_api_key_cache", user_api_key_cache) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "llm_router", _search_router()) + return user_api_key_cache + + +def _cache_team(cache: UserApiKeyCache, search_tools: list[str]) -> None: + team: Final = LiteLLM_TeamTableCachedObj( + team_id="team-1", + object_permission_id="op-team", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-team", search_tools=search_tools), + ) + cache.set_cache(key="team_id:team-1", value=team) + + +@pytest.mark.parametrize( + "general_settings, key_search_tools, team_search_tools, expected_status", + [ + ({}, [], [], 200), + ({"search_tool_deny_by_default": False}, [], [], 200), + ({"search_tool_deny_by_default": True}, ["search-a"], [], 403), + ({"search_tool_deny_by_default": True}, [], ["search-a"], 403), + ({"search_tool_deny_by_default": True}, ["search-a"], ["search-b"], 403), + ({"search_tool_deny_by_default": True}, ["search-a"], ["search-a"], 200), + ], +) +@pytest.mark.parametrize("path", ["/v1/search/search-a", "/search/search-a"]) +def test_direct_search_team_key_follows_search_tool_deny_by_default( + monkeypatch: pytest.MonkeyPatch, + cache: UserApiKeyCache, + tavily: respx.Route, + path: str, + general_settings: dict[str, bool], + key_search_tools: list[str], + team_search_tools: list[str], + expected_status: int, +): + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + _cache_team(cache, team_search_tools) + + response: Final = _client(_team_key(key_search_tools)).post(path, json={"query": "what is litellm"}) + + assert response.status_code == expected_status, response.text + if expected_status == 200: + assert response.json()["results"][0]["url"] == TAVILY_RESULT["url"] + assert tavily.call_count == 1 + else: + assert "search-a" in response.text + assert tavily.call_count == 0 + + +def test_direct_search_body_tool_name_is_denied_under_search_tool_deny_by_default(monkeypatch, cache, tavily): + monkeypatch.setattr(proxy_server, "general_settings", {"search_tool_deny_by_default": True}) + caller: Final = UserAPIKeyAuth(api_key="sk-standalone", user_role=LitellmUserRoles.INTERNAL_USER) + caller.via_virtual_key = True + + response: Final = _client(caller).post( + "/v1/search", json={"search_tool_name": "search-a", "query": "what is litellm"} + ) + + assert response.status_code == 403, response.text + assert response.json()["error"]["type"] == "key_search_tool_access_denied" + assert tavily.call_count == 0 def _json_request(body: dict[str, object]) -> MagicMock: @@ -52,3 +182,104 @@ async def test_search_with_only_a_query_falls_back_to_the_proxy_default_model(mo router.asearch.assert_awaited_once() assert router.asearch.await_args.kwargs["query"] == "litellm" assert router.asearch.await_args.kwargs["model"] == "perplexity-search" + + +def _standalone_key(key_search_tools: list[str] | None) -> UserAPIKeyAuth: + caller: Final = UserAPIKeyAuth( + api_key="sk-standalone", + user_role=LitellmUserRoles.INTERNAL_USER, + object_permission_id=None if key_search_tools is None else "op-key", + object_permission=( + None + if key_search_tools is None + else LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=key_search_tools) + ), + ) + caller.via_virtual_key = True + return caller + + +@pytest.mark.parametrize( + "general_settings, body", + [ + ({"search_tool_deny_by_default": True}, {"model": "search-a", "query": "what is litellm"}), + ( + {"search_tool_deny_by_default": True, "completion_model": "search-a"}, + {"query": "what is litellm"}, + ), + ], + ids=["body-model", "completion-model"], +) +@pytest.mark.parametrize("key_search_tools, expected_status", [(None, 403), (["search-a"], 200)]) +def test_direct_search_authorizes_the_tool_resolved_from_model_settings( + monkeypatch, cache, tavily, general_settings, body, key_search_tools, expected_status +): + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + + response: Final = _client(_standalone_key(key_search_tools)).post("/v1/search", json=body) + + assert response.status_code == expected_status, response.text + assert tavily.call_count == (1 if expected_status == 200 else 0) + + +@pytest.mark.parametrize( + "key_search_tools, expected_status, fallback_calls", + [(["search-a"], 500, 0), (["search-a", "search-b"], 200, 1)], +) +def test_router_search_fallback_target_must_be_granted( + monkeypatch, cache, key_search_tools, expected_status, fallback_calls +): + from litellm.proxy.auth.fallback_model_access import router_fallback_access_check + + monkeypatch.setattr(proxy_server, "general_settings", {"search_tool_deny_by_default": True}) + monkeypatch.setattr( # test-quality-ok: respx needs HTTPX enabled to fake the provider HTTP boundary. + litellm, + "disable_aiohttp_transport", + True, + ) + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[], + search_tools=[ + {"search_tool_name": "search-a", "litellm_params": {"search_provider": "exa_ai", "api_key": "fake"}}, + {"search_tool_name": "search-b", "litellm_params": {"search_provider": "tavily", "api_key": "fake"}}, + ], + fallbacks=[{"search-a": ["search-b"]}], + fallback_access_check=router_fallback_access_check, + num_retries=0, + ), + ) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=False) as mock: + failing_tool: Final = mock.post(url__regex=r"https://api\.exa\.ai/.*").respond(500, json={"error": "down"}) + fallback_tool: Final = mock.post(TAVILY_SEARCH_URL).respond(200, json={"results": [TAVILY_RESULT]}) + response: Final = _client(_standalone_key(key_search_tools)).post( + "/v1/search/search-a", json={"query": "what is litellm"} + ) + litellm.in_memory_llm_clients_cache.flush_cache() + + assert response.status_code == expected_status, response.text + assert failing_tool.call_count == 1 + assert fallback_tool.call_count == fallback_calls + + +@pytest.mark.parametrize( + "requested_tool, completion_model, expected_status", + [("search-b", "search-a", 403), ("search-a", "search-b", 200)], + ids=["requested-tool-ungranted", "completion-model-ungranted"], +) +def test_direct_search_authorizes_the_requested_tool_over_completion_model( + monkeypatch, cache, tavily, requested_tool, completion_model, expected_status +): + monkeypatch.setattr( + proxy_server, "general_settings", {"search_tool_deny_by_default": True, "completion_model": completion_model} + ) + + response: Final = _client(_standalone_key(["search-a"])).post( + f"/v1/search/{requested_tool}", json={"query": "what is litellm"} + ) + + assert response.status_code == expected_status, response.text + assert tavily.call_count == (1 if expected_status == 200 else 0) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c632ad01203..201746f1873 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30008,6 +30008,12 @@ export interface components { reject_clientside_metadata_tags?: boolean | null; /** @description Spreads the proxy's scheduled background jobs (spend flushes, budget resets, config reloads, exports) across a window instead of firing them together on every replica. On by default; set to tune the window, pin a job, or turn it off. */ scheduled_job_stagger?: components["schemas"]["ScheduledJobStaggerSettings"] | null; + /** + * Search Tool Deny By Default + * @description When True, a search tool must be explicitly listed in object_permission.search_tools: a virtual key needs its own grant plus its team's, a keyless team member needs the team's, and a user with neither needs their own. A missing permission record, an empty list, or an unresolved team grants nothing, and the unregistered search fallback is denied. The master key and dashboard sessions are exempt + * @default false + */ + search_tool_deny_by_default: boolean; /** @description Daily check of the spend LiteLLM captured against the provider's own bill (OpenAI via OPENAI_ADMIN_KEY). Publishes litellm_spend_capture_rate per provider and alerts when the ratio over the lookback window falls under the threshold (default 0.9). Off unless set. */ spend_capture_rate_check?: components["schemas"]["SpendCaptureRateCheckSettings"] | null; /** From 5ed7ec851148cbb47d33c95e9264f19a0bf4d8e6 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:55:26 -0700 Subject: [PATCH 4/5] fix(lens): price agent traces by joining gen_ai.response.id to spend logs (#44738) * fix(lens): price agent traces by joining gen_ai.response.id to spend logs Trace spend now joins each model call to spend_logs on one key: the span's response id (gen_ai.response.id or the id the normalizers read from OpenInference/LangChain output) against spend_logs.response_id or the upstream id embedded in a managed resp_ id. The litellm.call_id and traceparent transport join paths and the per-row ownership gate are removed; the spend SQL still restricts rows to what the reader can see. A run with some unpriced calls now reports the sum of its priced calls plus priced_calls, instead of an unknown total. * test(lens): cover response id spend join and partial trace totals * chore(lens): regenerate trace types for priced_calls * feat(lens): show partial run cost as a lower bound with priced call count * fix(lens): treat litellm.call_id as the same assigned call id for spend joins The id LiteLLM assigned to a call is either the response id it returned (gen_ai.response.id -> spend_logs.response_id) or its gateway call id (litellm.call_id -> spend_logs.litellm_call_id). Both are exact ids the gateway mints and logs, so the join stays one rule. Transport span matching and the ownership gate stay removed. * test(lens): cover litellm.call_id spend joins and restore captured totals * feat(lens): link each priced model call to its spend log Spans gain spend_log_request_id, the spend_logs.request_id the call was priced from, and spend_match, which says whether a model call matched or why not (no assigned id on the span, no spend log with that id, or an ambiguous match). A model call span is priced from the same ids as the run total, so its cost and the total agree. * test(lens): cover spend log links on model call spans * chore(lens): regenerate trace types for spend log links * feat(lens): open the matched spend log from an LLM step An LLM step's header now shows a Spend log chip with the matched request id and cost; clicking it opens the request log drawer over the run, fetched by the exact spend_logs.request_id instead of the span's own response id. Unpriced steps say why (no assigned id on the span, or no spend log with it). Tree rows show each model call's cost, and a partial run cost shows its priced call count inline. * test(lens): cover the spend log link and unmatched cost reasons * feat(lens): show the spend log link as a bordered LiteLLM Spend Log button * feat(lens): add a back link from the spend log drawer to the agent trace * feat(lens): label the spend log back link Back to Lens trace with the Lens icon * fix(lens): ignore assigned ids that name no spend log when pricing a call An id that names no row no longer vetoes the call, so a span carrying both a response id and a call id still prices from a spend row logged before litellm_call_id existed. An id naming two or more rows makes the call ambiguous, and the match reason comes from the same per-id result, so a single matched row with no cost is reported as matched. * fix(lens): price a trace only from spend logs in its own team A reader with several teams could see the same assigned id in another team's spend log; only rows from the trace's team now price it. The user and key ownership gate stays removed. * perf(lens): resolve each model call's spend once per trace Model call matches are computed once when the trace is resolved and looked up by span index, instead of scanning the model call list for every span and walking the graph again for spans, agents and the run total. * fix(lens): hide a step's Cost fact only when its spend log link shows the cost * chore(lens): drop narrative doc comments from the spend join * fix(lens): price a model call only when its ids agree on one spend log per span * fix(lens): keep pricing spend logs written before litellm_call_id by their request id * fix(lens): price every attempt a model call's ids name when they agree --- .../python_contract.json | 3 +- .../callbacks-legacy-python/src/adapter.rs | 7 +- .../src/test_support.rs | 2 +- litellm-rust/crates/traces-cache/src/cache.rs | 1 + litellm-rust/crates/traces-cache/src/spend.rs | 1 + .../crates/traces-cache/tests/read.rs | 2 + .../crates/traces-cache/tests/snapshots.rs | 1 + .../0016_spend_provider_request_id.sql | 4 + .../traces-clickhouse/query/spend_batch.sql | 5 +- .../query/spend_by_response_ids.sql | 5 +- .../query/trace_list_span_batch.sql | 2 +- .../query/trace_page_spans.sql | 2 +- .../query/trace_span_batch.sql | 2 +- .../traces-clickhouse/query/trace_spans.sql | 2 +- .../traces-clickhouse/src/query/named.rs | 11 +- .../crates/traces-clickhouse/src/span_row.rs | 5 +- .../traces-clickhouse/tests/migrations.rs | 5 + .../tests/queries/support.rs | 3 +- .../crates/traces-clickhouse/tests/reads.rs | 164 +- litellm-rust/crates/traces/src/lib.rs | 3 +- .../src/normalize/format/claude_code.rs | 4 +- .../src/normalize/instrumentation/mod.rs | 11 +- .../crates/traces/src/normalize/mod.rs | 48 +- litellm-rust/crates/traces/src/otlp/span.rs | 6 +- litellm-rust/crates/traces/src/query/named.rs | 15 + .../crates/traces/src/resolve/resolution.rs | 35 +- .../crates/traces/src/resolve/spend.rs | 106 +- .../crates/traces/src/resolve/view.rs | 40 +- litellm-rust/crates/traces/src/view.rs | 15 + litellm-rust/crates/traces/tests/captures.rs | 9 +- .../traces/tests/normalization_formats.rs | 25 + litellm-rust/crates/traces/tests/normalize.rs | 2 + .../crates/traces/tests/query/named.rs | 6 +- litellm-rust/crates/traces/tests/resolve.rs | 1850 ++++++++++------- .../clickhouse/clickhouse_spend_logger.py | 6 + litellm/litellm_core_utils/core_helpers.py | 4 +- .../llm_response_utils/get_headers.py | 18 +- litellm/llms/custom_httpx/llm_http_handler.py | 1 + .../rust_bridge/callbacks_legacy_python.py | 10 +- litellm/rust_bridge/trace/generated/types.py | 7 + litellm/tracing/types.py | 1 + .../trace_codegen/schemas/traces/Trace.json | 48 +- .../schemas/traces/TracePage.json | 9 +- .../test_get_headers.py | 15 + .../test_clickhouse_spend_logger.py | 13 + .../litellm_core_utils/test_core_helpers.py | 10 + .../custom_httpx/test_llm_http_handler.py | 7 +- .../rust_bridge/messages/test_route_host.py | 2 +- .../test_callbacks_legacy_python.py | 12 +- .../src/components/lens/data/demo/fixtures.ts | 5 + .../lens/data/demo/lensDemoLongTrace.ts | 2 + .../lens/traces/detail/run/RunHeader.tsx | 8 +- .../span/DetailPane.integration.test.tsx | 70 +- .../lens/traces/detail/span/PaneHeader.tsx | 24 +- .../lens/traces/detail/span/RequestTab.tsx | 45 +- .../lens/traces/detail/span/SpanPane.tsx | 7 +- .../lens/traces/detail/span/SpendLogLink.tsx | 75 + .../lens/traces/detail/tree/TreeRows.tsx | 6 + .../AgentTracesSection.integration.test.tsx | 2 +- .../traces/list/AgentTracesTable.test.tsx | 54 +- .../lens/traces/list/AgentTracesTable.tsx | 29 +- .../list/runSearch/__fixtures__/runs.ts | 1 + .../logs/detail/DrawerHeader.test.tsx | 36 + .../components/logs/detail/DrawerHeader.tsx | 18 +- .../logs/detail/LogDetailsDrawer.tsx | 5 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 + 66 files changed, 2098 insertions(+), 862 deletions(-) create mode 100644 litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/span/SpendLogLink.tsx diff --git a/litellm-rust/crates/callbacks-legacy-python/python_contract.json b/litellm-rust/crates/callbacks-legacy-python/python_contract.json index 9ed13ae5ed5..f65516265c5 100644 --- a/litellm-rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm-rust/crates/callbacks-legacy-python/python_contract.json @@ -88,7 +88,8 @@ "call_type" ], "stream_opened": [ - "logger" + "logger", + "head" ], "stream_success": [ "logger", diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index 21f563d9f3b..70b7ecb45fd 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -536,7 +536,7 @@ impl LegacyLogging { head.bind(py).set_item("cache_key", key)?; head.bind(py).set_item("cache_hit", true)?; } - Streaming::Opened.call(py, (self.logger()?.object(py),))?; + Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?; self.stream = Some(DeliveredStream { chunks: PyList::empty(py).unbind(), first_chunk: None, @@ -1768,13 +1768,13 @@ assert logger.calls[1][1] is response fn stream_bindings_deliver_collected_chunks_in_order_without_success_fan_out() { Python::initialize(); Python::attach(|py| { - let locals = namespace(py, c"first = b'first'\nlast = b'last'\nresponse = None"); + let locals = namespace(py, c"first = b'first'\nlast = b'last'\nresponse = None\nhead = {'additional_headers': {'request-id': 'req_native'}}"); let mut logging = LegacyLogging { operation: crate::LoggingOperation::Messages, ..logged(py, &locals, true) }; logging - .on_stream_open(py, &pyo3::types::PyDict::new(py).into_any().unbind()) + .on_stream_open(py, &local(&locals, "head").unbind()) .unwrap(); logging .on_stream_chunk(py, &local(&locals, "first").unbind()) @@ -1791,6 +1791,7 @@ assert logger.calls[1][1] is response &locals, c" assert logger.names() == ['stream_opened', 'stream_success'], logger.calls +assert logger.calls[0][1] is head chunks = logger.calls[1][1] assert len(chunks) == 2 assert chunks[0] is first diff --git a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs index 46c93369100..f252a45b562 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs @@ -84,7 +84,7 @@ FAKES = { 'success', response, call_type ), 'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type), - 'stream_opened': lambda logger: logger.record('stream_opened', None), + 'stream_opened': lambda logger, head: logger.record('stream_opened', head), 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record( 'stream_success', list(chunks) ), diff --git a/litellm-rust/crates/traces-cache/src/cache.rs b/litellm-rust/crates/traces-cache/src/cache.rs index f308714146e..a5526a39963 100644 --- a/litellm-rust/crates/traces-cache/src/cache.rs +++ b/litellm-rust/crates/traces-cache/src/cache.rs @@ -298,6 +298,7 @@ mod tests { fn row(span_id: &str) -> TraceSpansRow { TraceSpansRow { trace_id: String::new(), + original_trace_id: String::new(), span_id: span_id.into(), parent_span_id: String::new(), name: "run".into(), diff --git a/litellm-rust/crates/traces-cache/src/spend.rs b/litellm-rust/crates/traces-cache/src/spend.rs index 1cfcde841d0..6ea2cc79257 100644 --- a/litellm-rust/crates/traces-cache/src/spend.rs +++ b/litellm-rust/crates/traces-cache/src/spend.rs @@ -48,6 +48,7 @@ pub(super) async fn spend( let params = SpendByResponseIdsParams { access: access.clone(), response_ids: lookup.response_ids, + provider_request_ids: lookup.provider_request_ids, request_ids: lookup.request_ids, trace_ids: lookup.trace_ids, start_ms: window.start, diff --git a/litellm-rust/crates/traces-cache/tests/read.rs b/litellm-rust/crates/traces-cache/tests/read.rs index d714d7c9ed4..3f76d559a2b 100644 --- a/litellm-rust/crates/traces-cache/tests/read.rs +++ b/litellm-rust/crates/traces-cache/tests/read.rs @@ -260,6 +260,7 @@ fn access() -> ReadAccessParams { fn span(index: usize) -> TraceSpansRow { TraceSpansRow { trace_id: "trace".into(), + original_trace_id: String::new(), span_id: format!("span-{index}"), parent_span_id: if index == 0 { String::new() @@ -523,6 +524,7 @@ fn spend_row(response_id: &str, cost: f64) -> SpendByResponseIdsRow { litellm_call_id: String::new(), response_id: response_id.into(), upstream_response_id: String::new(), + provider_request_id: String::new(), trace_id: String::new(), span_id: String::new(), team_id: "team".into(), diff --git a/litellm-rust/crates/traces-cache/tests/snapshots.rs b/litellm-rust/crates/traces-cache/tests/snapshots.rs index 42085d3e95b..0a206004080 100644 --- a/litellm-rust/crates/traces-cache/tests/snapshots.rs +++ b/litellm-rust/crates/traces-cache/tests/snapshots.rs @@ -17,6 +17,7 @@ const TTL: Duration = Duration::from_secs(120); fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow { TraceSpansRow { trace_id: String::new(), + original_trace_id: String::new(), span_id: span_id.into(), parent_span_id: parent.into(), name: name.into(), diff --git a/litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql b/litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql new file mode 100644 index 00000000000..443ad0688e6 --- /dev/null +++ b/litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql @@ -0,0 +1,4 @@ +ALTER TABLE {database}.spend_logs + ADD COLUMN IF NOT EXISTS provider_request_id String DEFAULT '' AFTER response_id, + ADD INDEX IF NOT EXISTS idx_provider_request_id provider_request_id + TYPE bloom_filter(0.001) GRANULARITY 1 diff --git a/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql b/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql index 3918286a61f..55f8fe88a88 100644 --- a/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql @@ -1,5 +1,5 @@ SELECT * FROM ( -SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend, +SELECT request_id, litellm_call_id, response_id, upstream_response_id, provider_request_id, trace_id, span_id, team_id, api_key, user, spend, toUnixTimestamp64Milli(start_time) AS start_ms FROM ( SELECT *, @@ -15,7 +15,8 @@ FROM ( OR ({user_id:String} != '' AND user = {user_id:String}) OR has({team_ids:Array(String)}, team_id)) ) -WHERE response_id IN {response_ids:Array(String)} +WHERE provider_request_id IN {provider_request_ids:Array(String)} + OR response_id IN {response_ids:Array(String)} OR upstream_response_id IN {response_ids:Array(String)} OR litellm_call_id IN {request_ids:Array(String)} OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)}) diff --git a/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql b/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql index da64dafbc39..e6402fff6cf 100644 --- a/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql +++ b/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql @@ -1,4 +1,4 @@ -SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend, +SELECT request_id, litellm_call_id, response_id, upstream_response_id, provider_request_id, trace_id, span_id, team_id, api_key, user, spend, toUnixTimestamp64Milli(start_time) AS start_ms FROM ( SELECT *, @@ -14,7 +14,8 @@ FROM ( OR ({user_id:String} != '' AND user = {user_id:String}) OR has({team_ids:Array(String)}, team_id)) ) -WHERE response_id IN {response_ids:Array(String)} +WHERE provider_request_id IN {provider_request_ids:Array(String)} + OR response_id IN {response_ids:Array(String)} OR upstream_response_id IN {response_ids:Array(String)} OR litellm_call_id IN {request_ids:Array(String)} OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)}) diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql b/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql index 679edfdec2e..d6ab416dfd3 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql @@ -1,5 +1,5 @@ SELECT * FROM ( -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql b/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql index b81bba61e7c..27a8e2ac0da 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql @@ -1,4 +1,4 @@ -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql b/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql index 967ef2fcf1f..a084325feaa 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql @@ -1,5 +1,5 @@ SELECT * FROM ( -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql b/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql index 2e0ac4f6dfb..306692e9709 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql @@ -1,4 +1,4 @@ -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/src/query/named.rs b/litellm-rust/crates/traces-clickhouse/src/query/named.rs index beb4b42f76d..b7e70632480 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/named.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/named.rs @@ -82,6 +82,8 @@ pub use contracts::TraceSpansParams; struct TraceSpansRowEncoding { #[serde(default)] pub trace_id: String, + #[serde(default)] + pub original_trace_id: String, pub span_id: String, pub parent_span_id: String, pub name: String, @@ -181,6 +183,7 @@ struct SpendByResponseIdsParamsEncoding { #[serde(flatten)] pub access: contracts::ReadAccessParams, pub response_ids: Vec, + pub provider_request_ids: Vec, pub request_ids: Vec, pub trace_ids: Vec, #[serde(deserialize_with = "super::number::deserialize")] @@ -207,6 +210,8 @@ struct SpendByResponseIdsRowEncoding { pub litellm_call_id: String, pub response_id: String, pub upstream_response_id: String, + #[serde(default)] + pub provider_request_id: String, pub trace_id: String, pub span_id: String, pub team_id: String, @@ -344,7 +349,7 @@ mod tests { quoted, ); round_trip::( - json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), + json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), quoted, ); round_trip::( @@ -356,7 +361,7 @@ mod tests { quoted, ); round_trip::( - json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), + json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "provider_request_id": "req_provider", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), quoted, ); } @@ -374,7 +379,7 @@ mod tests { quoted, ); round_trip::( - json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), + json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "provider_request_ids": [], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), quoted, ); } diff --git a/litellm-rust/crates/traces-clickhouse/src/span_row.rs b/litellm-rust/crates/traces-clickhouse/src/span_row.rs index 0189dbff2c8..ad2863d6c5d 100644 --- a/litellm-rust/crates/traces-clickhouse/src/span_row.rs +++ b/litellm-rust/crates/traces-clickhouse/src/span_row.rs @@ -211,7 +211,10 @@ fn request_id(evidence: &CallEvidence) -> &str { .flatten() .find_map(|key| match key { CallKey::ProviderResponse(id) => Some(id.as_str()), - CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => None, + CallKey::LiteLlmRequest(_) + | CallKey::ProviderRequest(_) + | CallKey::Transport + | CallKey::GatewayAttempt => None, }) .unwrap_or_default() } diff --git a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs index d90c366118b..e51d9083c59 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs @@ -211,6 +211,10 @@ async fn schema_supports_span_rollups_and_spend_joins( Parameter::Strings(vec!["response-1".into()]), ), ("request_ids".into(), Parameter::Strings(Vec::new())), + ( + "provider_request_ids".into(), + Parameter::Strings(Vec::new()), + ), ("trace_ids".into(), Parameter::Strings(Vec::new())), ("all_teams".into(), Parameter::Integer(0)), ("user_id".into(), Parameter::Text(String::new())), @@ -2223,6 +2227,7 @@ async fn named_and_sql_readers_share_request_log_visibility( "api_key_hash": legacy_key.unwrap_or_default(), }))?, response_ids: vec!["shared-response".into()], + provider_request_ids: Vec::new(), request_ids: Vec::new(), trace_ids: Vec::new(), start_ms: timestamp / 1_000_000 - 1, diff --git a/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs b/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs index 371424c63d6..7532d9aecc9 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs @@ -158,7 +158,8 @@ fn span_row(span: &DecodedSpan, team: &str, key: &str) -> BTreeMap Some(id.as_str()), - litellm_traces::CallKey::Transport + litellm_traces::CallKey::ProviderRequest(_) + | litellm_traces::CallKey::Transport | litellm_traces::CallKey::GatewayAttempt => None, }) .unwrap_or_default() diff --git a/litellm-rust/crates/traces-clickhouse/tests/reads.rs b/litellm-rust/crates/traces-clickhouse/tests/reads.rs index e1996d6af04..b3d1b72bec1 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/reads.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/reads.rs @@ -601,7 +601,7 @@ async fn an_oversized_span_keeps_the_run_list_available_with_partial_totals( #[rstest] #[tokio::test] -async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback( +async fn assigned_call_ids_require_shared_ownership_through_detail_and_batch_reads( #[future(awt)] migrated_database: TestResult, ) -> TestResult { let fixture = migrated_database?; @@ -610,31 +610,41 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback let start_ms = 1_790_000_000_000_i64; let cases = [ ( - "gateway", - "provider-request", - "gateway", + "same-key", + "provider_response:same-key", "team-a", "key-a", Some(0.25), ), - ("legacy", "legacy", "", "team-a", "key-a", Some(0.25)), - ("conflict", "conflict", "different", "team-a", "key-a", None), + ( + "other-key", + "provider_response:other-key", + "team-a", + "key-b", + None, + ), ( "foreign-team", - "request", - "foreign-team", + "provider_response:foreign-team", "team-b", "key-a", None, ), ( - "foreign-key", - "request", - "foreign-key", + "call-id-other-key", + "litellm_request:call-id-other-key", "team-a", "key-b", None, ), + ( + "call-id-foreign-team", + "litellm_request:call-id-foreign-team", + "team-b", + "key-a", + None, + ), + ("transport-only", "transport:", "team-a", "key-a", None), ]; insert_rows( client, @@ -643,7 +653,7 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback InsertTable::OtelTraces, cases .iter() - .map(|(id, _, _, _, _, _)| { + .map(|(id, key, _, _, _)| { BTreeMap::from([ ("Timestamp".into(), json!(start_ms * 1_000_000)), ("Duration".into(), json!(1_000_000)), @@ -652,7 +662,7 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback ("ObservationType".into(), json!("llm")), ("TeamId".into(), json!("team-a")), ("ApiKeyHash".into(), json!("key-a")), - ("CallKeys".into(), json!([format!("litellm_request:{id}")])), + ("CallKeys".into(), json!([key])), ("CallEvidence".into(), json!("complete")), ]) }) @@ -666,11 +676,11 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback InsertTable::SpendLogs, cases .iter() - .map(|(_, request, call_id, team, key, _)| { + .map(|(id, _, team, key, _)| { BTreeMap::from([ - ("request_id".into(), json!(request)), - ("response_id".into(), json!("provider-response")), - ("litellm_call_id".into(), json!(call_id)), + ("request_id".into(), json!(format!("request-{id}"))), + ("response_id".into(), json!(id)), + ("litellm_call_id".into(), json!(id)), ("team_id".into(), json!(team)), ("api_key".into(), json!(key)), ("start_time".into(), json!(start_ms)), @@ -689,13 +699,13 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback let access = ReadAccessParams { all_teams: false, user_id: String::new(), - team_ids: vec!["team-a".into()], + team_ids: vec!["team-a".into(), "team-b".into()], }; let page = reader .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) .await?; assert_eq!(page.data.len(), cases.len()); - for (id, _, _, _, _, expected) in cases { + for (id, _, _, _, expected) in cases { let summary = page .data .iter() @@ -707,6 +717,122 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback .ok_or("missing trace")?; assert_eq!(detail.summary.spend, expected, "{id}"); assert_eq!(summary.spend, expected, "{id}"); + assert_eq!(summary.priced_calls, u64::from(expected.is_some()), "{id}"); } Ok(()) } + +#[rstest] +#[case::provider_request(true)] +#[case::transport(false)] +#[tokio::test] +async fn native_cost_correlation_survives_session_grouping_and_excludes_other_owners( + #[future(awt)] migrated_database: TestResult, + #[case] response_header: bool, + #[values(false, true)] grouped: bool, +) -> TestResult { + let fixture = migrated_database?; + let client = &fixture.database.client; + let writer = Connection::writer(&fixture.database.url)?; + let trace_id = if grouped { + "grouped-trace" + } else { + "original-trace" + }; + let start_ms = 1_790_000_000_000_i64; + let keys = if response_header { + vec!["provider_response:req_native"] + } else { + Vec::new() + }; + insert_rows( + client, + &writer, + DATABASE, + InsertTable::OtelTraces, + vec![BTreeMap::from([ + ("Timestamp".into(), json!(start_ms * 1_000_000)), + ("TraceId".into(), json!(trace_id)), + ("SpanId".into(), json!("native-call")), + ("SpanName".into(), json!("claude_code.llm_request")), + ("ObservationType".into(), json!("llm")), + ("Framework".into(), json!("claude-code")), + ("TeamId".into(), json!("team-a")), + ("ApiKeyHash".into(), json!("key-a")), + ("CallKeys".into(), json!(keys)), + ( + "CallEvidence".into(), + json!(if response_header { + "complete" + } else { + "unknown" + }), + ), + ( + "SpanAttributes".into(), + json!({"lens.original_trace_id": if grouped { "original-trace" } else { "" }}), + ), + ])], + ) + .await?; + insert_rows( + client, + &writer, + DATABASE, + InsertTable::SpendLogs, + ["key-a", "key-b"] + .into_iter() + .map(|key| { + BTreeMap::from([ + ("request_id".into(), json!(format!("log-{key}"))), + ("response_id".into(), json!("msg_native")), + ("provider_request_id".into(), json!("req_native")), + ( + "trace_id".into(), + json!(if response_header { + "" + } else { + "original-trace" + }), + ), + ( + "span_id".into(), + json!(if response_header { "" } else { "native-call" }), + ), + ("team_id".into(), json!("team-a")), + ("api_key".into(), json!(key)), + ("start_time".into(), json!(start_ms)), + ("end_time".into(), json!(start_ms + 1)), + ("spend".into(), json!(0.25)), + ]) + }) + .collect(), + ) + .await?; + let connection = fixture + .readers + .connection(client, &QueryScope::All, "fixture-secret") + .await?; + let (reader, store) = make_reader(client, connection); + let access = ReadAccessParams { + all_teams: true, + user_id: String::new(), + team_ids: Vec::new(), + }; + let page = reader + .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) + .await?; + assert_eq!(page.data.len(), 1); + let summary = &page.data[0]; + let detail = reader + .get_trace(&store, &access, trace_id, &summary.trace_ref) + .await? + .ok_or("missing trace")?; + assert_eq!((summary.spend, summary.priced_calls), (Some(0.25), 1)); + assert_eq!(detail.summary.spend, summary.spend); + assert_eq!( + detail.spans[0].spend_log_request_id.as_deref(), + Some("log-key-a") + ); + Ok(()) +} diff --git a/litellm-rust/crates/traces/src/lib.rs b/litellm-rust/crates/traces/src/lib.rs index f79fa59bdba..b51701638e8 100644 --- a/litellm-rust/crates/traces/src/lib.rs +++ b/litellm-rust/crates/traces/src/lib.rs @@ -44,5 +44,6 @@ pub use tenant::Tenant; pub use truncate::{truncate_messages, truncate_value}; pub use ui::{ChatRole, UiContent, UiField, UiMessage, UiToolCall, to_ui_content}; pub use view::{ - AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, Trace, TracePage, TraceSummary, + AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, SpendMatch, Trace, TracePage, + TraceSummary, }; diff --git a/litellm-rust/crates/traces/src/normalize/format/claude_code.rs b/litellm-rust/crates/traces/src/normalize/format/claude_code.rs index 18d7d74dc44..d24b9e6d8d2 100644 --- a/litellm-rust/crates/traces/src/normalize/format/claude_code.rs +++ b/litellm-rust/crates/traces/src/normalize/format/claude_code.rs @@ -6,7 +6,7 @@ use super::{Extraction, Format, SpanFacts}; use crate::{ Error, normalize::{ - CLAUDE_CODE_AGENT, CLAUDE_CODE_EVENTS_SCOPE, CLAUDE_CODE_SCOPE, CallEvidence, CallKey, + CLAUDE_CODE_AGENT, CLAUDE_CODE_EVENTS_SCOPE, CLAUDE_CODE_SCOPE, CallEvidence, ObservationType, RoleEvidence, SpanContext, attr, present, tokens, }, otlp::DecodedEvent, @@ -265,7 +265,7 @@ impl Format for ClaudeCode { output: llm_output(attributes), calls: present(attributes, &["gen_ai.response.id", "request_id"]) .map_or(CallEvidence::Unknown, |id| { - CallEvidence::complete(CallKey::ProviderResponse(id)) + CallEvidence::complete(crate::normalize::claude_call_key(id)) }), ..base }, diff --git a/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs b/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs index 550afae77ca..1003834de79 100644 --- a/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs +++ b/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs @@ -216,7 +216,16 @@ impl Instrumentation { fn with_call_ids(context: &SpanContext<'_>, facts: SpanFacts) -> SpanFacts { let calls = [ - present(context.attributes, &["gen_ai.response.id"]).map(CallKey::ProviderResponse), + present(context.attributes, &["gen_ai.response.id"]).map(|id| { + if matches!( + context.scope, + crate::normalize::CLAUDE_CODE_SCOPE | crate::normalize::CLAUDE_CODE_EVENTS_SCOPE + ) { + crate::normalize::claude_call_key(id) + } else { + CallKey::ProviderResponse(id) + } + }), present(context.attributes, &["litellm.call_id"]).map(CallKey::LiteLlmRequest), ] .into_iter() diff --git a/litellm-rust/crates/traces/src/normalize/mod.rs b/litellm-rust/crates/traces/src/normalize/mod.rs index f2c5376af10..53d5e521799 100644 --- a/litellm-rust/crates/traces/src/normalize/mod.rs +++ b/litellm-rust/crates/traces/src/normalize/mod.rs @@ -65,16 +65,26 @@ pub enum CallKey { LiteLlmRequest(String), /// The provider response id returned to the caller (`spend_logs.response_id`). ProviderResponse(String), + ProviderRequest(String), /// The span is the HTTP request itself; LiteLLM logs its `traceparent` span id. Transport, GatewayAttempt, } +pub(crate) fn claude_call_key(id: String) -> CallKey { + if id.starts_with("msg_") { + CallKey::ProviderResponse(id) + } else { + CallKey::ProviderRequest(id) + } +} + impl fmt::Display for CallKey { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::LiteLlmRequest(id) => write!(formatter, "litellm_request:{id}"), Self::ProviderResponse(id) => write!(formatter, "provider_response:{id}"), + Self::ProviderRequest(id) => write!(formatter, "provider_request:{id}"), Self::Transport => formatter.write_str("transport:"), Self::GatewayAttempt => formatter.write_str("gateway_attempt:"), } @@ -89,6 +99,9 @@ impl FromStr for CallKey { Some(("provider_response", id)) if !id.is_empty() => { Ok(Self::ProviderResponse(id.to_owned())) } + Some(("provider_request", id)) if !id.is_empty() => { + Ok(Self::ProviderRequest(id.to_owned())) + } Some(("litellm_request", id)) if !id.is_empty() => { Ok(Self::LiteLlmRequest(id.to_owned())) } @@ -127,24 +140,45 @@ pub enum CallEvidence { impl CallEvidence { pub(crate) fn row_keys(row: &crate::query::named::TraceSpansRow) -> BTreeSet { - if row.call_keys.is_empty() && !row.litellm_request_id.is_empty() { + let native = Self::native_request(row); + let keys = if row.call_keys.is_empty() && !row.litellm_request_id.is_empty() { BTreeSet::from([CallKey::ProviderResponse(row.litellm_request_id.clone())]) } else { row.call_keys.iter().cloned().collect() + }; + if !native { + return keys; } + if keys.is_empty() { + return BTreeSet::from([CallKey::Transport]); + } + keys.into_iter() + .map(|key| match key { + CallKey::ProviderResponse(id) => claude_call_key(id), + key => key, + }) + .collect() + } + + fn native_request(row: &crate::query::named::TraceSpansRow) -> bool { + matches!(row.framework.as_str(), "claude-code" | "claude-agent-sdk") + && row.name == "claude_code.llm_request" } pub(crate) fn from_row(row: &crate::query::named::TraceSpansRow) -> Self { - let kind = row - .call_evidence - .unwrap_or(if Self::row_keys(row).is_empty() { + let keys = Self::row_keys(row); + let kind = if Self::native_request(row) { + CallEvidenceKind::Complete + } else { + row.call_evidence.unwrap_or(if keys.is_empty() { CallEvidenceKind::Unknown } else { CallEvidenceKind::Complete - }); + }) + }; match kind { - CallEvidenceKind::Complete => Self::Complete(Self::row_keys(row)), - CallEvidenceKind::Partial => Self::Partial(Self::row_keys(row)), + CallEvidenceKind::Complete => Self::Complete(keys), + CallEvidenceKind::Partial => Self::Partial(keys), CallEvidenceKind::Unknown => Self::Unknown, } } diff --git a/litellm-rust/crates/traces/src/otlp/span.rs b/litellm-rust/crates/traces/src/otlp/span.rs index 4e63d78d286..58ccb9ea3b4 100644 --- a/litellm-rust/crates/traces/src/otlp/span.rs +++ b/litellm-rust/crates/traces/src/otlp/span.rs @@ -193,9 +193,9 @@ fn decoded_span( .into_iter() .flatten() .map(|key| match key { - crate::CallKey::LiteLlmRequest(id) | crate::CallKey::ProviderResponse(id) => { - id.len() + size_of::() - } + crate::CallKey::LiteLlmRequest(id) + | crate::CallKey::ProviderResponse(id) + | crate::CallKey::ProviderRequest(id) => id.len() + size_of::(), crate::CallKey::Transport | crate::CallKey::GatewayAttempt => { size_of::() } diff --git a/litellm-rust/crates/traces/src/query/named.rs b/litellm-rust/crates/traces/src/query/named.rs index a981bc46471..757012de16f 100644 --- a/litellm-rust/crates/traces/src/query/named.rs +++ b/litellm-rust/crates/traces/src/query/named.rs @@ -68,6 +68,8 @@ pub struct TraceSpansParams { pub struct TraceSpansRow { #[serde(default)] pub trace_id: String, + #[serde(default)] + pub original_trace_id: String, pub span_id: String, pub parent_span_id: String, pub name: String, @@ -113,6 +115,16 @@ pub struct TraceSpansRow { pub user_id: String, } +impl TraceSpansRow { + pub(crate) fn transport_trace_id(&self) -> &str { + if self.original_trace_id.is_empty() { + &self.trace_id + } else { + &self.original_trace_id + } + } +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct TracePageSpansParams { #[serde(flatten)] @@ -163,6 +175,7 @@ pub struct SpendByResponseIdsParams { #[serde(flatten)] pub access: ReadAccessParams, pub response_ids: Vec, + pub provider_request_ids: Vec, pub request_ids: Vec, pub trace_ids: Vec, pub start_ms: i64, @@ -175,6 +188,8 @@ pub struct SpendByResponseIdsRow { pub litellm_call_id: String, pub response_id: String, pub upstream_response_id: String, + #[serde(default)] + pub provider_request_id: String, pub trace_id: String, pub span_id: String, pub team_id: String, diff --git a/litellm-rust/crates/traces/src/resolve/resolution.rs b/litellm-rust/crates/traces/src/resolve/resolution.rs index 9b47755a16f..e19a98a1ede 100644 --- a/litellm-rust/crates/traces/src/resolve/resolution.rs +++ b/litellm-rust/crates/traces/src/resolve/resolution.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use indexmap::IndexMap; use crate::{ + SpendMatch, normalize::{CallKey, ObservationType}, query::named::{SpendByResponseIdsRow as SpendRow, TraceSpansRow}, }; @@ -27,8 +28,11 @@ pub(super) struct Resolution<'a> { types: HashMap<&'a str, ObservationType>, tool_failures: HashMap<&'a str, &'a TraceSpansRow>, pub(super) model_calls: Vec, + call_matches: HashMap>, } +pub(super) type CallMatch<'a> = (Option>, SpendMatch); + impl<'a> Resolution<'a> { pub(super) fn new(rows: &'a [TraceSpansRow], spend: &'a [SpendRow]) -> Self { let graph = Graph::new(rows); @@ -36,7 +40,7 @@ impl<'a> Resolution<'a> { let types: HashMap<&str, ObservationType> = (0..rows.len()) .map(|index| (graph.id(index), resolved_type(&graph, index, named_agents))) .collect(); - let model_calls = (0..rows.len()) + let model_calls: Vec = (0..rows.len()) .filter(|index| { types[graph.id(*index)] == ObservationType::Llm && !graph @@ -45,7 +49,7 @@ impl<'a> Resolution<'a> { .any(|descendant| types[graph.id(descendant)] == ObservationType::Llm) }) .collect(); - Self { + let resolution = Self { ownership: Ownership { team_id: &rows[0].team_id, api_key_hash: &rows[0].api_key_hash, @@ -65,6 +69,16 @@ impl<'a> Resolution<'a> { .map(|row| (row.tool_call_id.as_str(), row)) .collect(), model_calls, + call_matches: HashMap::new(), + }; + let call_matches = resolution + .model_calls + .iter() + .map(|call| (*call, resolution.resolve_call_match(*call))) + .collect(); + Self { + call_matches, + ..resolution } } @@ -118,7 +132,24 @@ impl<'a> Resolution<'a> { spend::requests(self.row(index), &self.ownership, self.spend) } + pub(super) fn call_match(&self, call: usize) -> Option<&CallMatch<'a>> { + self.call_matches.get(&call) + } + pub(super) fn call_requests(&self, call: usize) -> Option> { + self.call_match(call) + .and_then(|(requests, _)| requests.clone()) + } + + fn resolve_call_match(&self, call: usize) -> CallMatch<'a> { + if let Some(requests) = self.resolve_call_requests(call) { + return (Some(requests), SpendMatch::Matched); + } + let evidence = self.requests(call); + (None, evidence.unmatched_reason()) + } + + fn resolve_call_requests(&self, call: usize) -> Option> { let wrappers = self.graph.ancestors(call).into_iter().filter(|ancestor| { self.kind(*ancestor) == ObservationType::Llm && self diff --git a/litellm-rust/crates/traces/src/resolve/spend.rs b/litellm-rust/crates/traces/src/resolve/spend.rs index ce23362b7f8..a6b2fc0dbc5 100644 --- a/litellm-rust/crates/traces/src/resolve/spend.rs +++ b/litellm-rust/crates/traces/src/resolve/spend.rs @@ -3,7 +3,7 @@ use std::collections::BTreeSet; use indexmap::IndexMap; use crate::{ - CallEvidence, CallEvidenceKind, CallKey, + CallEvidence, CallEvidenceKind, CallKey, SpendMatch, query::named::{SpendByResponseIdsRow as SpendRow, TraceSpansRow}, }; @@ -12,6 +12,7 @@ use crate::{ pub struct SpendLookup { pub response_ids: Vec, pub request_ids: Vec, + pub provider_request_ids: Vec, /// Traces whose transport spans LiteLLM logged by `traceparent`. pub trace_ids: Vec, } @@ -45,13 +46,21 @@ impl SpendLookup { }) .collect(), ), + provider_request_ids: sorted( + keys() + .filter_map(|(_, key)| match key { + CallKey::ProviderRequest(id) => Some(id.clone()), + _ => None, + }) + .collect(), + ), trace_ids: sorted( keys() .filter_map(|(row, key)| match key { CallKey::Transport | CallKey::GatewayAttempt - if !row.trace_id.is_empty() => + if !row.transport_trace_id().is_empty() => { - Some(row.trace_id.clone()) + Some(row.transport_trace_id().to_owned()) } _ => None, }) @@ -61,7 +70,10 @@ impl SpendLookup { } pub fn is_empty(&self) -> bool { - self.response_ids.is_empty() && self.request_ids.is_empty() && self.trace_ids.is_empty() + self.response_ids.is_empty() + && self.request_ids.is_empty() + && self.provider_request_ids.is_empty() + && self.trace_ids.is_empty() } } @@ -86,6 +98,7 @@ pub(super) type Requests<'a> = Vec<&'a SpendRow>; enum KeyFamily { GatewayCall, ProviderResponse, + ProviderRequest, Transport, } @@ -93,12 +106,14 @@ fn key_family(key: &CallKey) -> KeyFamily { match key { CallKey::LiteLlmRequest(_) => KeyFamily::GatewayCall, CallKey::ProviderResponse(_) => KeyFamily::ProviderResponse, + CallKey::ProviderRequest(_) => KeyFamily::ProviderRequest, CallKey::Transport | CallKey::GatewayAttempt => KeyFamily::Transport, } } pub(super) enum KeyMatch<'a> { Missing, + Conflicting, Unique(&'a SpendRow), Ambiguous(Requests<'a>), } @@ -115,13 +130,13 @@ impl<'a> KeyMatch<'a> { fn unique(&self) -> Option<&'a SpendRow> { match self { Self::Unique(request) => Some(request), - Self::Missing | Self::Ambiguous(_) => None, + Self::Missing | Self::Conflicting | Self::Ambiguous(_) => None, } } fn agrees_with(&self, selected: &[&SpendRow]) -> bool { match self { - Self::Missing => false, + Self::Missing | Self::Conflicting => false, Self::Unique(request) => selected .iter() .any(|row| row.identity() == request.identity()), @@ -147,6 +162,29 @@ pub(super) enum SpendEvidence<'a> { } impl<'a> SpendEvidence<'a> { + pub(super) fn unmatched_reason(&self) -> SpendMatch { + match self { + Self::Unknown => SpendMatch::NoCallId, + Self::Partial(_) => SpendMatch::IncompleteEvidence, + Self::Complete(matches) if matches.is_empty() => SpendMatch::NoCallId, + Self::Complete(matches) + if matches.iter().any(|evidence| { + matches!(evidence, KeyMatch::Conflicting | KeyMatch::Ambiguous(_)) + }) => + { + SpendMatch::Ambiguous + } + Self::Complete(matches) + if matches + .iter() + .any(|evidence| matches!(evidence, KeyMatch::Missing)) => + { + SpendMatch::NoSpendLog + } + Self::Complete(_) => SpendMatch::Ambiguous, + } + } + pub(super) fn complete_requests(&self) -> Option> { match self { Self::Complete(matches) if !matches.is_empty() => { @@ -186,15 +224,16 @@ fn matches<'a>( CallKey::ProviderResponse(id) => { !id.is_empty() && (spend.response_id == *id || spend.upstream_response_id == *id) } + CallKey::ProviderRequest(id) => !id.is_empty() && spend.provider_request_id == *id, CallKey::LiteLlmRequest(id) => { !id.is_empty() && (spend.litellm_call_id == *id || (spend.litellm_call_id.is_empty() && spend.request_id == *id)) } CallKey::Transport | CallKey::GatewayAttempt => { - !row.trace_id.is_empty() + !row.transport_trace_id().is_empty() && !row.span_id.is_empty() - && spend.trace_id == row.trace_id + && spend.trace_id == row.transport_trace_id() && spend.span_id == row.span_id } }; @@ -253,7 +292,8 @@ pub(super) fn requests<'a>( let matches = aliases .into_iter() .map(|(_, requests)| { - KeyMatch::new( + let had_candidates = !requests.is_empty(); + let matched = KeyMatch::new( requests .into_iter() .filter(|request| { @@ -262,7 +302,12 @@ pub(super) fn requests<'a>( .all(|family| family.contains(&request.identity())) }) .collect(), - ) + ); + if had_candidates && matches!(matched, KeyMatch::Missing) { + KeyMatch::Conflicting + } else { + matched + } }) .collect(); match evidence.kind() { @@ -280,18 +325,35 @@ pub(super) fn request_cost(requests: &[&SpendRow]) -> Option { }) } -pub(super) fn total(calls: &[Option>]) -> Option { - if calls.is_empty() { - return None; - } - let requests: Option> = calls - .iter() - .map(|requests| requests.as_ref()) - .collect::>>() - .map(|calls| calls.into_iter().flatten().copied().collect()); - let unique: IndexMap<(&str, i64, &str), &SpendRow> = requests? +pub(super) fn unique<'a>(requests: impl IntoIterator) -> Requests<'a> { + requests .into_iter() .map(|request| (request.identity(), request)) - .collect(); - request_cost(&unique.into_values().collect::>()) + .collect::>() + .into_values() + .collect() +} + +pub(super) struct Priced { + pub(super) spend: Option, + pub(super) priced_calls: u64, +} + +pub(super) fn total(calls: &[Option>]) -> Priced { + let priced: Vec<&Requests<'_>> = calls + .iter() + .flatten() + .filter(|requests| request_cost(requests).is_some()) + .collect(); + let spend = if priced.is_empty() { + None + } else { + request_cost(&unique( + priced.iter().flat_map(|requests| requests.iter().copied()), + )) + }; + Priced { + spend, + priced_calls: priced.len() as u64, + } } diff --git a/litellm-rust/crates/traces/src/resolve/view.rs b/litellm-rust/crates/traces/src/resolve/view.rs index 87688d9b36e..9e9edd51bf4 100644 --- a/litellm-rust/crates/traces/src/resolve/view.rs +++ b/litellm-rust/crates/traces/src/resolve/view.rs @@ -6,7 +6,7 @@ use time::OffsetDateTime; use crate::{ normalize::ObservationType, query::named::{ListTracesRow, SpendByResponseIdsRow as SpendRow, TraceSpansRow}, - view::{AgentNode, Span, SpanStatus, Trace, TraceSummary}, + view::{AgentNode, Span, SpanStatus, SpendMatch, Trace, TraceSummary}, }; use super::{ @@ -23,7 +23,18 @@ fn optional(value: &str) -> Option { fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span { let row = resolution.row(index); let status = resolution.status_source(index); - let requests = resolution.requests(index).complete_requests(); + let (requests, spend_match) = if let Some((requests, matched)) = resolution.call_match(index) { + (requests.clone(), Some(*matched)) + } else { + (resolution.requests(index).complete_requests(), None) + }; + let spend = requests + .as_ref() + .and_then(|requests| request_cost(requests)); + let spend_log_request_id = match (spend_match, requests.as_deref()) { + (Some(SpendMatch::Matched), Some([request])) => Some(request.request_id.clone()), + _ => None, + }; Span { span_id: row.span_id.clone(), parent_span_id: optional(&row.parent_span_id), @@ -42,9 +53,9 @@ fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span input_tokens: row.input_tokens, output_tokens: row.output_tokens, litellm_request_id: optional(&row.litellm_request_id), - spend: requests - .as_ref() - .and_then(|requests| request_cost(requests)), + spend, + spend_log_request_id, + spend_match, } } @@ -87,6 +98,7 @@ fn agents(resolution: &Resolution<'_>) -> Vec { .filter(|(owner, _)| *owner == name) .map(|(_, requests)| requests.clone()) .collect(); + let priced = total(&owned_calls); AgentNode { name: name.to_owned(), parent_agent, @@ -101,7 +113,8 @@ fn agents(resolution: &Resolution<'_>) -> Vec { .map(|span| graph.rows[*span].duration_ns) .sum::() as f64 / NANOS_PER_MS, - spend: total(&owned_calls), + spend: priced.spend, + priced_calls: priced.priced_calls, } }) .collect() @@ -160,6 +173,12 @@ pub fn resolve_trace( } else { calls.iter().map(|call| &rows[*call]).collect() }; + let priced = total( + &calls + .iter() + .map(|call| resolution.call_requests(*call)) + .collect::>(), + ); let first_input = spans .iter() .zip(rows) @@ -208,12 +227,8 @@ pub fn resolve_trace( input_tokens: counted.iter().map(|row| u64::from(row.input_tokens)).sum(), output_tokens: counted.iter().map(|row| u64::from(row.output_tokens)).sum(), models: sorted_unique(calls.iter().map(|call| rows[*call].model.as_str())), - spend: total( - &calls - .iter() - .map(|call| resolution.call_requests(*call)) - .collect::>(), - ), + spend: priced.spend, + priced_calls: priced.priced_calls, }; Some(Trace { summary, @@ -250,5 +265,6 @@ pub fn listed_summary(row: &ListTracesRow) -> TraceSummary { output_tokens: row.output_tokens, models: row.models.clone(), spend: None, + priced_calls: 0, } } diff --git a/litellm-rust/crates/traces/src/view.rs b/litellm-rust/crates/traces/src/view.rs index a864c740526..5a214d33da7 100644 --- a/litellm-rust/crates/traces/src/view.rs +++ b/litellm-rust/crates/traces/src/view.rs @@ -37,6 +37,19 @@ pub struct Span { pub output_tokens: u32, pub litellm_request_id: Option, pub spend: Option, + pub spend_log_request_id: Option, + pub spend_match: Option, +} + +#[macro_rules_attribute::apply(response_type)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum SpendMatch { + Matched, + NoCallId, + NoSpendLog, + Ambiguous, + IncompleteEvidence, } /// One distinct agent in a trace: 200 invocations of `researcher` are one node. @@ -50,6 +63,7 @@ pub struct AgentNode { pub tool_calls: u64, pub duration_ms: f64, pub spend: Option, + pub priced_calls: u64, } #[macro_rules_attribute::apply(response_type)] @@ -80,6 +94,7 @@ pub struct TraceSummary { pub output_tokens: u64, pub models: Vec, pub spend: Option, + pub priced_calls: u64, } #[macro_rules_attribute::apply(response_type)] diff --git a/litellm-rust/crates/traces/tests/captures.rs b/litellm-rust/crates/traces/tests/captures.rs index ca498fc914a..ff20db617f6 100644 --- a/litellm-rust/crates/traces/tests/captures.rs +++ b/litellm-rust/crates/traces/tests/captures.rs @@ -117,6 +117,7 @@ fn captured_spend_rows(spend_logs: &str) -> (FixtureCapture, Vec TraceSpansRow { .into_iter() .flatten() .find_map(|key| match key { - CallKey::ProviderResponse(id) => Some(id.clone()), + CallKey::ProviderResponse(id) | CallKey::ProviderRequest(id) => Some(id.clone()), CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => None, }) .unwrap_or_default(); TraceSpansRow { trace_id: span.trace_id, + original_trace_id: String::new(), span_id: span.span_id, parent_span_id: span.parent_span_id, name: span.name, @@ -267,6 +269,7 @@ fn unrelated_transport(call: &TraceSpansRow) -> TraceSpansRow { .expect("valid unrelated transport timestamp"); TraceSpansRow { trace_id: call.trace_id.clone(), + original_trace_id: call.original_trace_id.clone(), span_id: format!("unrelated-transport-{}", call.span_id), parent_span_id: call.parent_span_id.clone(), name: "unrelated-http".into(), @@ -405,7 +408,9 @@ fn redundant_genai_response_id_keeps_call_evidence( let response_ids: Vec<_> = keys .iter() .filter_map(|key| match key { - CallKey::ProviderResponse(id) => Some(id.clone()), + CallKey::ProviderResponse(id) | CallKey::ProviderRequest(id) => { + Some(id.clone()) + } CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => { None } diff --git a/litellm-rust/crates/traces/tests/normalization_formats.rs b/litellm-rust/crates/traces/tests/normalization_formats.rs index 124f03593c8..e8b11ece64a 100644 --- a/litellm-rust/crates/traces/tests/normalization_formats.rs +++ b/litellm-rust/crates/traces/tests/normalization_formats.rs @@ -928,3 +928,28 @@ fn convention_markers_keep_genai_call_evidence( let marked = decode(span, scope, &marked_attributes, vec![]).unwrap(); assert_eq!(marked.normalized.calls, plain.normalized.calls); } + +#[rstest] +#[case::request("req_native", CallKey::ProviderRequest("req_native".into()))] +#[case::legacy_message("msg_legacy", CallKey::ProviderResponse("msg_legacy".into()))] +fn native_claude_preserves_the_provider_id_family( + span: Span, + #[case] id: &str, + #[case] key: CallKey, +) { + let native = Span { + name: "claude_code.llm_request".into(), + ..span + }; + let decoded = decode( + native, + "com.anthropic.claude_code.tracing", + &[("gen_ai.response.id", id)], + Vec::new(), + ) + .unwrap(); + assert_eq!( + decoded.normalized.calls, + CallEvidence::Complete(std::collections::BTreeSet::from([key])) + ); +} diff --git a/litellm-rust/crates/traces/tests/normalize.rs b/litellm-rust/crates/traces/tests/normalize.rs index 923e2006088..bfff82fcdd7 100644 --- a/litellm-rust/crates/traces/tests/normalize.rs +++ b/litellm-rust/crates/traces/tests/normalize.rs @@ -284,6 +284,7 @@ fn llamaindex_wrapped_responses_keep_provider_call_keys(#[case] body: &[u8]) { #[rstest] #[case::request(litellm_traces::CallKey::LiteLlmRequest("request:with:colons".to_owned()))] #[case::response(litellm_traces::CallKey::ProviderResponse("response:with:colons".to_owned()))] +#[case::provider_request(litellm_traces::CallKey::ProviderRequest("req_native".into()))] #[case::transport(litellm_traces::CallKey::Transport)] #[case::gateway_attempt(litellm_traces::CallKey::GatewayAttempt)] fn call_keys_round_trip_through_storage(#[case] key: litellm_traces::CallKey) { @@ -305,6 +306,7 @@ fn call_keys_round_trip_through_storage(#[case] key: litellm_traces::CallKey) { #[rstest] #[case::missing_separator("provider_response")] #[case::missing_response("provider_response:")] +#[case::missing_provider_request("provider_request:")] #[case::missing_request("litellm_request:")] #[case::transport_id("transport:unexpected")] #[case::gateway_attempt_separator("gateway_attempt")] diff --git a/litellm-rust/crates/traces/tests/query/named.rs b/litellm-rust/crates/traces/tests/query/named.rs index 6b853ebf151..3ac9668e42a 100644 --- a/litellm-rust/crates/traces/tests/query/named.rs +++ b/litellm-rust/crates/traces/tests/query/named.rs @@ -43,7 +43,7 @@ fn named_requests_preserve_all_access_cases( json!({"trace_id": "trace", "trace_ref": "ref", "span_id": "span", "error_offset": u64::MAX, "error_version": "version"}), )); round_trip::(request( - json!({"response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), + json!({"response_ids": ["response"], "provider_request_ids": [], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), )); } @@ -53,7 +53,7 @@ fn result_contracts_preserve_public_field_names() { json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "STATUS_CODE_OK", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "agent_names": ["agent"], "frameworks": ["framework"], "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}), ); round_trip::( - json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), + json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), ); round_trip::( json!({"span_id": "span", "input": "input", "output": "output", "attributes": {"count": "42"}}), @@ -62,6 +62,6 @@ fn result_contracts_preserve_public_field_names() { json!({"span_id": "span", "message": "error", "total_chars": u64::MAX, "version": "version"}), ); round_trip::( - json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), + json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "provider_request_id": "req_provider", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), ); } diff --git a/litellm-rust/crates/traces/tests/resolve.rs b/litellm-rust/crates/traces/tests/resolve.rs index 641d15db5b9..8d65adb0c0f 100644 --- a/litellm-rust/crates/traces/tests/resolve.rs +++ b/litellm-rust/crates/traces/tests/resolve.rs @@ -1,5 +1,5 @@ use litellm_traces::{ - AgentNode, SpanStatus, iso_time, listed_summary, + AgentNode, SpanStatus, SpendMatch, iso_time, listed_summary, query::named::{ListTracesRow, SpendByResponseIdsRow, TraceSpansRow}, resolve_trace, }; @@ -11,6 +11,7 @@ const MS: i64 = 1_000_000; fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow { TraceSpansRow { trace_id: String::new(), + original_trace_id: String::new(), span_id: span_id.into(), parent_span_id: parent.into(), name: name.into(), @@ -32,8 +33,8 @@ fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> Trac call_keys: Vec::new(), call_evidence: None, tool_call_id: String::new(), - team_id: String::new(), - api_key_hash: String::new(), + team_id: "team".into(), + api_key_hash: "key".into(), user_id: String::new(), } } @@ -55,31 +56,18 @@ fn llm(span_id: &str, parent: &str, agent: &str, response_id: &str) -> TraceSpan } } -fn owned(mut span: TraceSpansRow, team: &str, user: &str, key: &str) -> TraceSpansRow { - span.team_id = team.into(); - span.user_id = user.into(); - span.api_key_hash = key.into(); - span -} - -fn spend( - request_id: &str, - response_id: &str, - team: &str, - user: &str, - key: &str, - cost: f64, -) -> SpendByResponseIdsRow { +fn spend(request_id: &str, response_id: &str, cost: f64) -> SpendByResponseIdsRow { SpendByResponseIdsRow { request_id: request_id.into(), - response_id: response_id.into(), litellm_call_id: String::new(), + response_id: response_id.into(), upstream_response_id: String::new(), + provider_request_id: String::new(), trace_id: String::new(), span_id: String::new(), - team_id: team.into(), - api_key: key.into(), - user: user.into(), + api_key: "key".into(), + user: String::new(), + team_id: "team".into(), spend: Some(cost), start_ms: T0 / MS, } @@ -221,6 +209,7 @@ fn repeated_subagent_invocations_aggregate_into_one_node() { tool_calls: 1, duration_ms: 1000.0, spend: None, + priced_calls: 0, } ); let researcher = &trace.agents[1]; @@ -384,38 +373,15 @@ fn names_and_frameworks_are_sorted_and_distinct() { } #[rstest] -fn repeated_response_counts_once_and_other_owners_are_ignored() { +fn repeated_response_id_counts_once() { let rows = [ - owned( - row("root", "", "agent", "agent", "agent"), - "team-a", - "", - "key-a", - ), - owned( - llm("llm-1", "root", "agent", "response-1"), - "team-a", - "", - "key-a", - ), - owned( - llm("llm-2", "root", "agent", "response-1"), - "team-a", - "", - "key-a", - ), + row("root", "", "agent", "agent", "agent"), + llm("llm-1", "root", "agent", "response-1"), + llm("llm-2", "root", "agent", "response-1"), ]; let spend = [ - spend("request-other", "response-1", "team-b", "", "key-b", 99.0), - spend("request-1", "response-1", "team-a", "", "key-a", 0.25), - spend( - "request-other-key", - "unrelated-response", - "team-a", - "", - "key-c", - 50.0, - ), + spend("request-1", "response-1", 0.25), + spend("request-other", "unrelated-response", 50.0), ]; let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); assert_eq!(trace.summary.spend, Some(0.25)); @@ -431,352 +397,363 @@ fn repeated_response_counts_once_and_other_owners_are_ignored() { } #[rstest] -fn ambiguous_response_id_keeps_cost_unknown() { - let rows = [owned( - llm("llm-1", "", "agent", "response-1"), - "", - "user", - "key-a", - )]; - let spend = [ - spend("response-1", "response-1", "", "user", "key-a", 0.25), - spend( - "response-1_cache_hit123", - "response-1", - "", - "user", - "key-a", - 0.0, - ), - ]; - let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); - assert_eq!((trace.summary.spend, trace.spans[0].spend), (None, None)); -} - -#[rstest] -#[case::key_differs("team", "", "export", "team", "", "request", false)] -#[case::shared_key("team", "", "export", "team", "", "export", true)] -#[case::shared_user("", "user", "export", "", "user", "request", true)] -#[case::teamless_key("", "", "key", "", "", "key", true)] -#[case::other_team("team", "user", "key", "other-team", "user", "key", false)] -#[case::other_user("", "user", "export", "", "other-user", "request", false)] -#[case::no_shared_identity("", "", "export", "", "", "request", false)] -#[case::no_identity("", "", "", "", "", "", false)] -#[case::master_key_without_spend_key("", "", "master", "", "", "", false)] -fn cost_requires_shared_ownership( - #[case] trace_team: &str, - #[case] trace_user: &str, - #[case] trace_key: &str, - #[case] spend_team: &str, - #[case] spend_user: &str, - #[case] spend_key: &str, - #[case] known: bool, -) { +fn model_calls_link_the_spend_log_they_were_priced_from() { let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - trace_team, - trace_user, - trace_key, - ), - owned( - llm("llm", "agent", "agent", "response"), - trace_team, - trace_user, - trace_key, - ), + row("agent", "", "agent", "agent", "agent"), + llm("matched", "agent", "agent", "matched"), + llm("no-log", "agent", "agent", "missing"), + llm("no-id", "agent", "agent", ""), + llm("ambiguous", "agent", "agent", "cached"), ]; - let spend = [spend( - "request", "response", spend_team, spend_user, spend_key, 0.25, - )]; - let trace = resolve_trace("trace", "visible-reference", &rows, &spend).unwrap(); - let expected = known.then_some(0.25); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); - assert_eq!(trace.spans[1].spend, expected); -} - -#[rstest] -#[case::missing_id("missing_id")] -#[case::missing_spend("missing_spend")] -#[case::duplicate_spend("duplicate_spend")] -fn incomplete_call_cost_never_becomes_a_partial_total(#[case] failure: &str) { - let second_id = if failure == "missing_id" { - "" - } else { - "second" - }; - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "export", - ), - owned( - llm("first", "agent", "agent", "first"), - "team", - "", - "export", - ), - owned( - llm("second", "agent", "agent", second_id), - "team", - "", - "export", - ), + let logs = [ + spend("request-matched", "matched", 0.25), + spend("request-cached-a", "cached", 0.25), + spend("request-cached-b", "cached", 0.0), ]; - let first = spend("first", "first", "team", "", "export", 0.25); - let second = spend("second", "second", "team", "", "export", 0.25); - let duplicate = spend("duplicate", "second", "team", "", "export", 0.25); - let spend = if failure == "duplicate_spend" { - vec![first, second, duplicate] - } else { - vec![first] - }; - let trace = resolve_trace("trace", "ref", &rows, &spend).unwrap(); - assert_eq!(trace.spans[1].spend, Some(0.25)); - assert_eq!(trace.spans[2].spend, None); - assert_eq!(trace.summary.spend, None); - assert_eq!(trace.agents[0].spend, None); -} - -#[rstest] -fn transport_spans_complete_a_call_without_its_own_id() { - let mut transport = row("http", "llm", "POST", "framework", ""); - transport.trace_id = "trace".into(); - transport.call_keys = vec!["transport:".parse().unwrap()]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - let mut call = llm("llm", "agent", "agent", ""); - call.trace_id = "trace".into(); - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(call, "team", "", "key"), - owned(transport, "team", "", "key"), - ]; - let mut logged = spend("request", "", "team", "", "key", 0.5); - logged.trace_id = "trace".into(); - logged.span_id = "http".into(); - let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); - assert_eq!(trace.summary.spend, Some(0.5)); -} - -#[rstest] -#[case::lone_call(1, Some(0.5))] -#[case::two_calls(2, None)] -fn sibling_transports_belong_to_the_only_model_call_under_their_parent( - #[case] calls: usize, - #[case] expected: Option, -) { - let mut transport = at( - row("http", "step", "gateway.request", "framework", ""), - 2, - 10, + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + let links: Vec<_> = trace + .spans + .iter() + .map(|span| (span.spend_log_request_id.as_deref(), span.spend_match)) + .collect(); + assert_eq!( + links, + [ + (None, None), + (Some("request-matched"), Some(SpendMatch::Matched)), + (None, Some(SpendMatch::NoSpendLog)), + (None, Some(SpendMatch::NoCallId)), + (None, Some(SpendMatch::Ambiguous)), + ] ); - transport.trace_id = "trace".into(); - transport.call_keys = vec![litellm_traces::CallKey::GatewayAttempt]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - let mut rows = vec![ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), - owned(transport, "team", "", "key"), - ]; - for index in 0..calls { - let mut call = llm(&format!("chat-{index}"), "step", "agent", ""); - call.call_evidence = None; - rows.push(owned(call, "team", "", "key")); - } - let mut logged = spend("request", "", "team", "", "key", 0.5); - logged.trace_id = "trace".into(); - logged.span_id = "http".into(); - let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); } #[rstest] -#[case::without_tool_http_sibling(None, false, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::after_call(Some((200, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::inside_call_without_spend(Some((10, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::inside_call_with_unrelated_spend(Some((10, 10)), true, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::missing_gateway_attempt(Some((10, 10)), false, litellm_traces::CallKey::GatewayAttempt, None)] -fn sibling_transport_does_not_lose_model_call_spend( - #[case] transport_timing: Option<(i64, u64)>, - #[case] unrelated_spend: bool, +fn model_call_span_cost_agrees_with_the_run_total_when_priced_from_a_wrapper() { + let rows = [ + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("wrapper", "", "agent", "") + }, + llm("call", "wrapper", "agent", ""), + ]; + let logs = [SpendByResponseIdsRow { + litellm_call_id: "gateway".into(), + ..spend("request", "chatcmpl-request", 0.25) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!( + ( + trace.spans[1].spend, + trace.spans[1].spend_log_request_id.as_deref() + ), + (Some(0.25), Some("request")) + ); +} + +#[rstest] +fn unpriced_call_leaves_a_partial_total_of_the_priced_calls() { + let rows = [ + row("agent", "", "agent", "agent", "agent"), + llm("first", "agent", "agent", "first"), + llm("second", "agent", "agent", "second"), + llm("third", "agent", "agent", ""), + ]; + let trace = resolve_trace("trace", "ref", &rows, &[spend("first", "first", 0.25)]).unwrap(); + assert_eq!( + ( + trace.summary.spend, + trace.summary.priced_calls, + trace.summary.llm_calls + ), + (Some(0.25), 1, 3) + ); + assert_eq!( + (trace.agents[0].spend, trace.agents[0].priced_calls), + (Some(0.25), 1) + ); + assert_eq!( + trace + .spans + .iter() + .map(|span| span.spend) + .collect::>(), + [None, Some(0.25), None, None] + ); +} + +#[rstest] +fn no_priced_call_leaves_cost_unknown() { + let rows = [llm("call", "", "agent", "response")]; + let trace = resolve_trace("trace", "ref", &rows, &[spend("other", "other", 0.25)]).unwrap(); + assert_eq!((trace.summary.spend, trace.summary.priced_calls), (None, 0)); +} + +fn gateway_logged(request_id: &str, call_id: &str, cost: f64) -> SpendByResponseIdsRow { + SpendByResponseIdsRow { + litellm_call_id: call_id.into(), + ..spend(request_id, &format!("chatcmpl-{request_id}"), cost) + } +} + +#[rstest] +#[case::response_id(litellm_traces::CallKey::ProviderResponse("chatcmpl-request".into()), Some(0.25))] +#[case::gateway_call_id(litellm_traces::CallKey::LiteLlmRequest("gateway".into()), Some(0.25))] +#[case::other_call_id(litellm_traces::CallKey::LiteLlmRequest("other".into()), None)] +#[case::request_id_is_not_a_call_id(litellm_traces::CallKey::LiteLlmRequest("request".into()), None)] +#[case::transport(litellm_traces::CallKey::Transport, None)] +#[case::gateway_attempt(litellm_traces::CallKey::GatewayAttempt, None)] +fn only_ids_litellm_assigned_join_spend( #[case] key: litellm_traces::CallKey, #[case] expected: Option, ) { - let call = owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: vec![litellm_traces::CallKey::ProviderResponse( - "chatcmpl-1".into(), - )], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("chat", "step", "agent", "chatcmpl-1") - }, - "team", - "", - "key", + let rows = [TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec![key], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [SpendByResponseIdsRow { + upstream_response_id: "trace".into(), + ..gateway_logged("request", "gateway", 0.25) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + (trace.summary.spend, trace.summary.priced_calls), + (expected, u64::from(expected.is_some())) ); - let base_rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), - call, - ]; - let rows: Vec<_> = base_rows - .into_iter() - .chain(transport_timing.map(|(start, duration)| { - let mut transport = at( - row("tool-http", "step", "GET", "framework", ""), - start, - duration, - ); - transport.trace_id = "trace".into(); - transport.call_keys = vec![key]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - owned(transport, "team", "", "key") - })) - .collect(); - let logs: Vec<_> = std::iter::once(spend("chatcmpl-1", "chatcmpl-1", "team", "", "key", 0.5)) - .chain(unrelated_spend.then(|| SpendByResponseIdsRow { - trace_id: "trace".into(), - span_id: "tool-http".into(), - ..spend("unrelated", "unrelated", "team", "", "key", 0.75) - })) - .collect(); - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); } #[rstest] -#[case::agreeing_ids( - litellm_traces::CallKey::Transport, - "call-a", - Some("response-a"), - Some(0.25) -)] -#[case::conflicting_gateway_id(litellm_traces::CallKey::Transport, "call-b", None, None)] -#[case::conflicting_response_id( - litellm_traces::CallKey::Transport, - "call-a", - Some("response-b"), - None -)] -#[case::conflicting_gateway_and_response( - litellm_traces::CallKey::Transport, - "call-b", - Some("response-b"), - None -)] -#[case::agreeing_gateway_attempt( - litellm_traces::CallKey::GatewayAttempt, - "call-a", - Some("response-a"), - Some(0.25) -)] -#[case::conflicting_gateway_attempt(litellm_traces::CallKey::GatewayAttempt, "call-b", None, None)] -fn gateway_attempt_identifiers_must_match_one_spend_row( - #[case] transport: litellm_traces::CallKey, - #[case] call_id: &str, - #[case] response_id: Option<&str>, +#[case::same_team("team", Some(0.25), SpendMatch::Matched)] +#[case::other_team("other-team", None, SpendMatch::NoSpendLog)] +fn spend_logs_price_only_calls_in_their_own_team( + #[case] logged_team: &str, #[case] expected: Option, + #[case] matched: SpendMatch, ) { - let keys = [ - transport, - litellm_traces::CallKey::LiteLlmRequest(call_id.into()), - ] - .into_iter() - .chain(response_id.map(|id| litellm_traces::CallKey::ProviderResponse(id.into()))) - .collect(); - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(llm("call", "agent", "agent", ""), "team", "", "key"), - owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: keys, - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..row("attempt", "call", "gateway.request", "framework", "") - }, - "team", - "", - "key", - ), - ]; - let logs = [ - SpendByResponseIdsRow { - litellm_call_id: "call-a".into(), - trace_id: "trace".into(), - span_id: "attempt".into(), - ..spend("request-a", "response-a", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - litellm_call_id: "call-b".into(), - trace_id: "trace".into(), - span_id: "other-attempt".into(), - ..spend("request-b", "response-b", "team", "", "key", 0.5) - }, - ]; + let rows = [TraceSpansRow { + team_id: "team".into(), + ..llm("call", "", "agent", "response") + }]; + let logs = [SpendByResponseIdsRow { + team_id: logged_team.into(), + ..spend("request", "response", 0.25) + }]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); - assert_eq!(trace.spans[2].spend, expected); + assert_eq!( + (trace.summary.spend, trace.spans[0].spend_match), + (expected, Some(matched)) + ); } #[rstest] -#[case::legacy_row("", Some(0.5))] -#[case::other_call("other-call", None)] -fn gateway_id_miss_only_vetoes_rows_that_carry_a_call_id( +#[case::legacy_row_without_call_id("", Some(0.25), SpendMatch::Matched)] +#[case::call_id_names_nothing("missing", None, SpendMatch::Ambiguous)] +fn recorded_gateway_id_must_agree_unless_spend_predates_gateway_ids( #[case] logged_call_id: &str, #[case] expected: Option, + #[case] matched: SpendMatch, ) { - let mut transport = row("http", "llm", "gateway.request", "framework", ""); - transport.trace_id = "trace".into(); - transport.call_keys = vec![ - "transport:".parse().unwrap(), - "litellm_request:gateway-call".parse().unwrap(), - ]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - let mut call = llm("llm", "agent", "agent", ""); - call.trace_id = "trace".into(); - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-request".into()), + litellm_traces::CallKey::LiteLlmRequest("gateway".into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [gateway_logged("request", logged_call_id, 0.25)]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + (trace.spans[0].spend, trace.spans[0].spend_match), + (expected, Some(matched)) + ); +} + +#[rstest] +#[case::missing_cost(None, SpendMatch::Matched)] +#[case::finite_cost(Some(0.25), SpendMatch::Matched)] +fn a_single_matched_row_is_matched_whatever_its_cost( + #[case] cost: Option, + #[case] matched: SpendMatch, +) { + let rows = [llm("call", "", "agent", "response")]; + let logs = [SpendByResponseIdsRow { + spend: cost, + ..spend("request", "response", 0.0) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + ( + trace.spans[0].spend, + trace.spans[0].spend_match, + trace.spans[0].spend_log_request_id.as_deref() ), - owned(call, "team", "", "key"), - owned(transport, "team", "", "key"), + (cost, Some(matched), Some("request")) + ); + assert_eq!(trace.summary.priced_calls, u64::from(cost.is_some())); +} + +#[rstest] +#[case::same_request("gateway", Some(0.25), SpendMatch::Matched)] +#[case::conflicting_requests("other", None, SpendMatch::Ambiguous)] +fn response_id_and_call_id_must_name_the_same_request( + #[case] call_id: &str, + #[case] expected: Option, + #[case] matched: SpendMatch, +) { + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-request".into()), + litellm_traces::CallKey::LiteLlmRequest(call_id.into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [ + gateway_logged("request", "gateway", 0.25), + gateway_logged("other-request", "other", 0.5), ]; - let mut logged = spend("chatcmpl-1", "chatcmpl-1", "team", "", "key", 0.5); - logged.trace_id = "trace".into(); - logged.span_id = "http".into(); - logged.litellm_call_id = logged_call_id.into(); - let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); - assert_eq!(trace.summary.spend, expected); + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + ( + trace.spans[0].spend, + trace.summary.spend, + trace.spans[0].spend_match + ), + (expected, expected, Some(matched)) + ); +} + +#[rstest] +fn call_id_picks_the_call_when_its_response_id_has_a_cache_hit_twin() { + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-shared".into()), + litellm_traces::CallKey::LiteLlmRequest("gateway".into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [ + SpendByResponseIdsRow { + litellm_call_id: "gateway".into(), + ..spend("request", "chatcmpl-shared", 0.25) + }, + SpendByResponseIdsRow { + litellm_call_id: "cache-hit".into(), + ..spend("request_cache_hit", "chatcmpl-shared", 0.0) + }, + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + ( + trace.summary.spend, + trace.spans[0].spend_log_request_id.as_deref() + ), + (Some(0.25), Some("request")) + ); +} + +#[rstest] +fn wrapper_recording_a_retry_and_its_final_attempt_prices_both() { + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-retry".into()), + litellm_traces::CallKey::ProviderResponse("chatcmpl-final".into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [ + spend("retry", "chatcmpl-retry", 0.25), + spend("final", "chatcmpl-final", 0.5), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + (trace.summary.spend, trace.spans[0].spend_match), + (Some(0.75), Some(SpendMatch::Matched)) + ); +} + +#[rstest] +fn call_id_prices_a_legacy_spend_log_without_litellm_call_id() { + let rows = [TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("legacy".into())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [spend("legacy", "chatcmpl-legacy", 0.25)]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); +} + +#[rstest] +fn spend_lookup_collects_only_assigned_ids() { + let recorded = TraceSpansRow { + trace_id: "trace".to_owned(), + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("response".to_owned()), + litellm_traces::CallKey::LiteLlmRequest("request".to_owned()), + litellm_traces::CallKey::Transport, + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Unknown), + ..row("span", "", "operation", "llm", "") + }; + assert_eq!( + litellm_traces::SpendLookup::new(&[recorded]), + litellm_traces::SpendLookup { + response_ids: vec!["response".into()], + provider_request_ids: Vec::new(), + request_ids: vec!["request".into()], + trace_ids: vec!["trace".into()], + } + ); +} + +#[rstest] +fn upstream_response_id_inside_a_managed_id_joins_spend() { + let rows = [llm("call", "", "agent", "chatcmpl-upstream")]; + let logs = [SpendByResponseIdsRow { + upstream_response_id: "chatcmpl-upstream".into(), + ..spend("request", "resp_managed", 0.25) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); +} + +#[rstest] +fn conflicting_complete_wrapper_cannot_price_an_unrelated_call() { + let rows = [ + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::ProviderResponse( + "retry-response".into(), + )], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("wrapper", "", "agent", "") + }, + llm("call", "wrapper", "agent", "response"), + ]; + let logs = [ + spend("retry", "retry-response", 0.25), + spend("final", "response", 0.5), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!((trace.summary.spend, trace.summary.priced_calls), (None, 0)); +} + +#[rstest] +fn ambiguous_response_id_keeps_cost_unknown() { + let rows = [llm("llm-1", "", "agent", "response-1")]; + let spend = [ + spend("response-1", "response-1", 0.25), + spend("response-1_cache_hit123", "response-1", 0.0), + ]; + let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); + assert_eq!((trace.summary.spend, trace.spans[0].spend), (None, None)); } #[rstest] @@ -827,37 +804,6 @@ fn iso_time_matches_python_isoformat(#[case] ms: i64, #[case] expected: &str) { assert_eq!(iso_time(ms), expected); } -#[rstest] -#[case::narrows_ambiguity("request-a", Some(0.25))] -#[case::conflicting_exact_request("request-c", None)] -fn complete_wrapper_reconciles_ambiguous_response( - #[case] exact_id: &str, - #[case] expected: Option, -) { - let wrapper = TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest(exact_id.to_owned())], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..owned(llm("wrapper", "", "agent", ""), "team", "", "key") - }; - let rows = [ - wrapper, - owned( - llm("call", "wrapper", "agent", "response"), - "team", - "", - "key", - ), - ]; - let logs = [ - spend("request-a", "response", "team", "", "key", 0.25), - spend("request-b", "response", "team", "", "key", 0.5), - spend("request-c", "other-response", "team", "", "key", 0.75), - ]; - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); -} - #[rstest] #[case::missing(None, None)] #[case::free(Some(0.0), Some(0.0))] @@ -868,15 +814,10 @@ fn complete_correlation_requires_known_finite_cost( #[case] cost: Option, #[case] expected: Option, ) { - let rows = [owned( - llm("call", "", "agent", "response"), - "team", - "", - "key", - )]; + let rows = [llm("call", "", "agent", "response")]; let logged = SpendByResponseIdsRow { spend: cost, - ..spend("request", "response", "team", "", "key", 0.25) + ..spend("request", "response", 0.25) }; let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); assert_eq!(trace.summary.spend, expected); @@ -884,160 +825,39 @@ fn complete_correlation_requires_known_finite_cost( assert_eq!(trace.spans[0].spend, expected); } -#[rstest] -#[case::complete_retry(true, litellm_traces::CallEvidenceKind::Complete, Some(0.75))] -#[case::missing_retry(false, litellm_traces::CallEvidenceKind::Complete, None)] -#[case::unknown_retry(true, litellm_traces::CallEvidenceKind::Unknown, None)] -#[case::partial_retry(true, litellm_traces::CallEvidenceKind::Partial, None)] -fn transports_preserve_retry_spend_without_counting_unrelated_cached_rows( - #[case] retry_logged: bool, - #[case] retry_evidence: litellm_traces::CallEvidenceKind, - #[case] expected: Option, -) { - let transport = |id: &str| { - owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: vec!["transport:".parse().unwrap()], - call_evidence: Some(if id == "first" { - retry_evidence - } else { - litellm_traces::CallEvidenceKind::Complete - }), - ..row(id, "call", "POST", "framework", "") - }, - "team", - "", - "key", - ) - }; - let rows = [ - owned( - llm("call", "", "agent", "final-response"), - "team", - "", - "key", - ), - transport("first"), - transport("second"), - ]; - let logs = [ - SpendByResponseIdsRow { - trace_id: "trace".into(), - span_id: "first".into(), - ..spend("retry", "retry-response", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - trace_id: "trace".into(), - span_id: "second".into(), - ..spend("final", "final-response", "team", "", "key", 0.5) - }, - spend("cached", "final-response", "team", "", "key", 0.0), - ]; - let available = if retry_logged { &logs[..] } else { &logs[1..] }; - let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); -} - -#[rstest] -#[case::same_request(false)] -#[case::ambiguous_response(true)] -fn multiple_identifiers_for_one_request_count_its_spend_once(#[case] cached_row: bool) { - let rows = [owned( - TraceSpansRow { - call_keys: vec![ - "provider_response:response".parse().unwrap(), - "litellm_request:request".parse().unwrap(), - ], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("call", "", "agent", "response") - }, - "team", - "", - "key", - )]; - let logs = [ - spend("request", "response", "team", "", "key", 0.25), - spend("cached", "response", "team", "", "key", 0.5), - ]; - let available = if cached_row { &logs[..] } else { &logs[..1] }; - let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); - assert_eq!(trace.summary.spend, Some(0.25)); - assert_eq!(trace.agents[0].spend, Some(0.25)); - assert_eq!(trace.spans[0].spend, Some(0.25)); -} - #[rstest] #[case::finite(0.25, Some(0.5))] #[case::overflow(f64::MAX, None)] fn trace_cost_requires_a_finite_total(#[case] cost: f64, #[case] expected: Option) { let rows = [ - owned(llm("first", "", "agent", "response-a"), "team", "", "key"), - owned(llm("second", "", "agent", "response-b"), "team", "", "key"), + llm("first", "", "agent", "response-a"), + llm("second", "", "agent", "response-b"), ]; let logs = [ - spend("request-a", "response-a", "team", "", "key", cost), - spend("request-b", "response-b", "team", "", "key", cost), + spend("request-a", "response-a", cost), + spend("request-b", "response-b", cost), ]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); assert_eq!(trace.summary.spend, expected); assert_eq!(trace.agents[0].spend, expected); } -#[rstest] -fn complete_wrapper_accounts_for_retries_missing_from_the_call_span() { - let rows = [ - owned( - TraceSpansRow { - call_keys: vec![ - "litellm_request:retry".parse().unwrap(), - "litellm_request:final".parse().unwrap(), - ], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("wrapper", "", "agent", "") - }, - "team", - "", - "key", - ), - owned( - llm("call", "wrapper", "agent", "response"), - "team", - "", - "key", - ), - ]; - let logs = [ - spend("retry", "retry-response", "team", "", "key", 0.25), - spend("final", "response", "team", "", "key", 0.5), - ]; - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, Some(0.75)); - assert_eq!(trace.agents[0].spend, Some(0.75)); -} - #[rstest] #[case::legacy(None, Some(0.25))] #[case::unknown(Some(litellm_traces::CallEvidenceKind::Unknown), None)] #[case::partial(Some(litellm_traces::CallEvidenceKind::Partial), None)] #[case::complete(Some(litellm_traces::CallEvidenceKind::Complete), Some(0.25))] -fn legacy_request_id_fallback_respects_recorded_evidence( +fn stored_response_id_requires_complete_call_evidence( #[case] evidence: Option, #[case] expected: Option, ) { - let span = owned( - TraceSpansRow { - call_evidence: evidence, - ..llm("call", "", "agent", "response") - }, - "team", - "", - "key", - ); + let span = TraceSpansRow { + call_evidence: evidence, + ..llm("call", "", "agent", "response") + }; let stored = serde_json::to_value(span).unwrap(); let decoded: TraceSpansRow = serde_json::from_value(stored).unwrap(); - let logs = [spend("request", "response", "team", "", "key", 0.25)]; + let logs = [spend("request", "response", 0.25)]; let trace = resolve_trace("trace", "ref", &[decoded], &logs).unwrap(); assert_eq!(trace.summary.spend, expected); assert_eq!(trace.spans[0].spend, expected); @@ -1058,28 +878,6 @@ fn malformed_stored_span_fields_are_rejected( assert!(serde_json::from_value::(encoded).is_err()); } -#[rstest] -#[case::unknown(litellm_traces::CallEvidenceKind::Unknown)] -#[case::complete(litellm_traces::CallEvidenceKind::Complete)] -fn spend_lookup_fetches_recorded_keys_before_resolving_completeness( - #[case] evidence: litellm_traces::CallEvidenceKind, -) { - let recorded = TraceSpansRow { - trace_id: "trace".to_owned(), - call_keys: vec![ - litellm_traces::CallKey::ProviderResponse("response".to_owned()), - litellm_traces::CallKey::LiteLlmRequest("request".to_owned()), - litellm_traces::CallKey::Transport, - ], - call_evidence: Some(evidence), - ..row("span", "", "operation", "llm", "") - }; - let lookup = litellm_traces::SpendLookup::new(&[recorded]); - assert_eq!(lookup.response_ids, ["response"]); - assert_eq!(lookup.request_ids, ["request"]); - assert_eq!(lookup.trace_ids, ["trace"]); -} - #[rstest] #[case::parent_first(false)] #[case::child_first(true)] @@ -1150,68 +948,23 @@ fn empty_root_preview_uses_the_earliest_agent_or_model_input() { fn repeated_request_ids_preserve_storage_identity(#[case] reverse: bool) { let first = SpendByResponseIdsRow { start_ms: 100, - ..spend("same", "response", "team", "", "key", 0.25) + ..spend("same", "response", 0.25) }; let second = SpendByResponseIdsRow { start_ms: 200, - ..spend("same", "response", "team", "", "key", 0.5) + ..spend("same", "response", 0.5) }; let logs = if reverse { [second, first] } else { [first, second] }; - let rows = [owned( - llm("call", "", "agent", "response"), - "team", - "", - "key", - )]; + let rows = [llm("call", "", "agent", "response")]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); assert_eq!(trace.summary.spend, None); assert_eq!(trace.spans[0].spend, None); } -#[rstest] -#[case::gateway(false)] -#[case::transport(true)] -fn independent_key_disambiguates_repeated_request_ids(#[case] transport: bool) { - let rows = [owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: vec![ - litellm_traces::CallKey::ProviderResponse("response".into()), - if transport { - litellm_traces::CallKey::Transport - } else { - litellm_traces::CallKey::LiteLlmRequest("gateway".into()) - }, - ], - ..llm("call", "", "agent", "response") - }, - "team", - "", - "key", - )]; - let logs = [ - SpendByResponseIdsRow { - start_ms: 100, - litellm_call_id: "gateway".into(), - trace_id: "trace".into(), - span_id: "call".into(), - ..spend("same", "response", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - start_ms: 200, - litellm_call_id: "other".into(), - ..spend("same", "response", "team", "", "key", 0.5) - }, - ]; - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, Some(0.25)); - assert_eq!(trace.spans[0].spend, Some(0.25)); -} - #[rstest] #[case::same_row(true, Some(0.25))] #[case::distinct_rows(false, Some(0.75))] @@ -1220,27 +973,19 @@ fn totals_deduplicate_only_equal_storage_identities( #[case] expected: Option, ) { let rows = [ - owned(llm("first", "", "agent", "a"), "team", "", "key"), - owned( - llm("second", "", "agent", if duplicate { "a" } else { "b" }), - "team", - "", - "key", - ), + llm("first", "", "agent", "a"), + llm("second", "", "agent", if duplicate { "a" } else { "b" }), ]; let logs = [ SpendByResponseIdsRow { start_ms: 100, - ..spend("same", "a", "team", "", "key", 0.25) + ..spend("same", "a", 0.25) }, SpendByResponseIdsRow { start_ms: if duplicate { 100 } else { 200 }, ..spend( "same", if duplicate { "a" } else { "b" }, - "team", - "", - "key", if duplicate { 0.25 } else { 0.5 }, ) }, @@ -1254,80 +999,6 @@ fn totals_deduplicate_only_equal_storage_identities( ); } -#[rstest] -fn conflicting_keys_cannot_agree_on_request_id_alone() { - let rows = [ - owned( - TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], - ..llm("wrapper", "", "agent", "") - }, - "team", - "", - "key", - ), - owned( - llm("call", "wrapper", "agent", "response"), - "team", - "", - "key", - ), - ]; - let logs = [ - SpendByResponseIdsRow { - start_ms: 100, - ..spend("same", "response", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - start_ms: 200, - litellm_call_id: "gateway".into(), - ..spend("same", "other", "team", "", "key", 0.5) - }, - ]; - assert_eq!( - resolve_trace("trace", "ref", &rows, &logs) - .unwrap() - .summary - .spend, - None - ); -} - -#[rstest] -#[case::gateway("gateway", "provider-id", "team", "key", Some(0.25))] -#[case::legacy("", "gateway", "team", "key", Some(0.25))] -#[case::conflict("other", "gateway", "team", "key", None)] -#[case::other_team("gateway", "provider-id", "other-team", "key", None)] -#[case::other_key("gateway", "provider-id", "team", "other-key", None)] -fn gateway_lookup_respects_legacy_fallback_and_ownership( - #[case] call_id: &str, - #[case] request_id: &str, - #[case] team: &str, - #[case] key: &str, - #[case] expected: Option, -) { - let rows = [owned( - TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], - ..llm("call", "", "agent", "") - }, - "team", - "", - "key", - )]; - let logs = [SpendByResponseIdsRow { - litellm_call_id: call_id.into(), - ..spend(request_id, "provider", team, "", key, 0.25) - }]; - assert_eq!( - resolve_trace("trace", "ref", &rows, &logs) - .unwrap() - .summary - .spend, - expected - ); -} - #[rstest] #[case::matching("call-one", "claude_code.tool.execution", SpanStatus::Error)] #[case::other_tool("other-call", "claude_code.tool.execution", SpanStatus::Ok)] @@ -1389,3 +1060,790 @@ fn native_tool_failure_log_matches_by_call_id_without_double_counting( assert_eq!(trace.spans[0].status, expected); assert_eq!(trace.summary.error_count, 1); } + +fn owned_spend( + request_id: &str, + response_id: &str, + team: &str, + user: &str, + key: &str, + cost: f64, +) -> SpendByResponseIdsRow { + SpendByResponseIdsRow { + request_id: request_id.into(), + response_id: response_id.into(), + litellm_call_id: String::new(), + upstream_response_id: String::new(), + provider_request_id: String::new(), + trace_id: String::new(), + span_id: String::new(), + team_id: team.into(), + api_key: key.into(), + user: user.into(), + spend: Some(cost), + start_ms: T0 / MS, + } +} + +fn owned(mut span: TraceSpansRow, team: &str, user: &str, key: &str) -> TraceSpansRow { + span.team_id = team.into(); + span.user_id = user.into(); + span.api_key_hash = key.into(); + span +} + +#[rstest] +fn repeated_response_counts_once_and_other_owners_are_ignored() { + let rows = [ + owned( + row("root", "", "agent", "agent", "agent"), + "team-a", + "", + "key-a", + ), + owned( + llm("llm-1", "root", "agent", "response-1"), + "team-a", + "", + "key-a", + ), + owned( + llm("llm-2", "root", "agent", "response-1"), + "team-a", + "", + "key-a", + ), + ]; + let spend = [ + owned_spend("request-other", "response-1", "team-b", "", "key-b", 99.0), + owned_spend("request-1", "response-1", "team-a", "", "key-a", 0.25), + owned_spend( + "request-other-key", + "unrelated-response", + "team-a", + "", + "key-c", + 50.0, + ), + ]; + let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!(trace.agents[0].spend, Some(0.25)); + assert_eq!( + trace + .spans + .iter() + .map(|span| span.spend) + .collect::>(), + [None, Some(0.25), Some(0.25)] + ); +} + +#[rstest] +#[case::key_differs("team", "", "export", "team", "", "request", false)] +#[case::shared_key("team", "", "export", "team", "", "export", true)] +#[case::shared_user("", "user", "export", "", "user", "request", true)] +#[case::teamless_key("", "", "key", "", "", "key", true)] +#[case::other_team("team", "user", "key", "other-team", "user", "key", false)] +#[case::other_user("", "user", "export", "", "other-user", "request", false)] +#[case::no_shared_identity("", "", "export", "", "", "request", false)] +#[case::no_identity("", "", "", "", "", "", false)] +#[case::master_key_without_spend_key("", "", "master", "", "", "", false)] +fn cost_requires_shared_ownership( + #[case] trace_team: &str, + #[case] trace_user: &str, + #[case] trace_key: &str, + #[case] spend_team: &str, + #[case] spend_user: &str, + #[case] spend_key: &str, + #[case] known: bool, +) { + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + trace_team, + trace_user, + trace_key, + ), + owned( + llm("llm", "agent", "agent", "response"), + trace_team, + trace_user, + trace_key, + ), + ]; + let spend = [owned_spend( + "request", "response", spend_team, spend_user, spend_key, 0.25, + )]; + let trace = resolve_trace("trace", "visible-reference", &rows, &spend).unwrap(); + let expected = known.then_some(0.25); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); + assert_eq!(trace.spans[1].spend, expected); +} + +#[rstest] +#[case::missing_id("missing_id")] +#[case::missing_spend("missing_spend")] +#[case::duplicate_spend("duplicate_spend")] +fn partial_trace_total_counts_only_complete_calls(#[case] failure: &str) { + let second_id = if failure == "missing_id" { + "" + } else { + "second" + }; + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "export", + ), + owned( + llm("first", "agent", "agent", "first"), + "team", + "", + "export", + ), + owned( + llm("second", "agent", "agent", second_id), + "team", + "", + "export", + ), + ]; + let first = owned_spend("first", "first", "team", "", "export", 0.25); + let second = owned_spend("second", "second", "team", "", "export", 0.25); + let duplicate = owned_spend("duplicate", "second", "team", "", "export", 0.25); + let spend = if failure == "duplicate_spend" { + vec![first, second, duplicate] + } else { + vec![first] + }; + let trace = resolve_trace("trace", "ref", &rows, &spend).unwrap(); + assert_eq!(trace.spans[1].spend, Some(0.25)); + assert_eq!(trace.spans[2].spend, None); + assert_eq!( + (trace.summary.spend, trace.summary.priced_calls), + (Some(0.25), 1) + ); + assert_eq!( + (trace.agents[0].spend, trace.agents[0].priced_calls), + (Some(0.25), 1) + ); +} + +#[rstest] +fn transport_spans_complete_a_call_without_its_own_id() { + let mut transport = row("http", "llm", "POST", "framework", ""); + transport.trace_id = "trace".into(); + transport.call_keys = vec!["transport:".parse().unwrap()]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + let mut call = llm("llm", "agent", "agent", ""); + call.trace_id = "trace".into(); + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(call, "team", "", "key"), + owned(transport, "team", "", "key"), + ]; + let mut logged = owned_spend("request", "", "team", "", "key", 0.5); + logged.trace_id = "trace".into(); + logged.span_id = "http".into(); + let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); + assert_eq!(trace.summary.spend, Some(0.5)); +} + +#[rstest] +#[case::lone_call(1, Some(0.5))] +#[case::two_calls(2, None)] +fn sibling_transports_belong_to_the_only_model_call_under_their_parent( + #[case] calls: usize, + #[case] expected: Option, +) { + let mut transport = at( + row("http", "step", "gateway.request", "framework", ""), + 2, + 10, + ); + transport.trace_id = "trace".into(); + transport.call_keys = vec![litellm_traces::CallKey::GatewayAttempt]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + let mut rows = vec![ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), + owned(transport, "team", "", "key"), + ]; + for index in 0..calls { + let mut call = llm(&format!("chat-{index}"), "step", "agent", ""); + call.call_evidence = None; + rows.push(owned(call, "team", "", "key")); + } + let mut logged = owned_spend("request", "", "team", "", "key", 0.5); + logged.trace_id = "trace".into(); + logged.span_id = "http".into(); + let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::without_tool_http_sibling(None, false, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::after_call(Some((200, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::inside_call_without_spend(Some((10, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::inside_call_with_unrelated_spend(Some((10, 10)), true, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::missing_gateway_attempt(Some((10, 10)), false, litellm_traces::CallKey::GatewayAttempt, None)] +fn sibling_transport_does_not_lose_model_call_spend( + #[case] transport_timing: Option<(i64, u64)>, + #[case] unrelated_spend: bool, + #[case] key: litellm_traces::CallKey, + #[case] expected: Option, +) { + let call = owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec![litellm_traces::CallKey::ProviderResponse( + "chatcmpl-1".into(), + )], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("chat", "step", "agent", "chatcmpl-1") + }, + "team", + "", + "key", + ); + let base_rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), + call, + ]; + let rows: Vec<_> = base_rows + .into_iter() + .chain(transport_timing.map(|(start, duration)| { + let mut transport = at( + row("tool-http", "step", "GET", "framework", ""), + start, + duration, + ); + transport.trace_id = "trace".into(); + transport.call_keys = vec![key]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + owned(transport, "team", "", "key") + })) + .collect(); + let logs: Vec<_> = std::iter::once(owned_spend( + "chatcmpl-1", + "chatcmpl-1", + "team", + "", + "key", + 0.5, + )) + .chain(unrelated_spend.then(|| SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "tool-http".into(), + ..owned_spend("unrelated", "unrelated", "team", "", "key", 0.75) + })) + .collect(); + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::agreeing_ids( + litellm_traces::CallKey::Transport, + "call-a", + Some("response-a"), + Some(0.25) +)] +#[case::conflicting_gateway_id(litellm_traces::CallKey::Transport, "call-b", None, None)] +#[case::conflicting_response_id( + litellm_traces::CallKey::Transport, + "call-a", + Some("response-b"), + None +)] +#[case::conflicting_gateway_and_response( + litellm_traces::CallKey::Transport, + "call-b", + Some("response-b"), + None +)] +#[case::agreeing_gateway_attempt( + litellm_traces::CallKey::GatewayAttempt, + "call-a", + Some("response-a"), + Some(0.25) +)] +#[case::conflicting_gateway_attempt(litellm_traces::CallKey::GatewayAttempt, "call-b", None, None)] +fn gateway_attempt_identifiers_must_match_one_spend_row( + #[case] transport: litellm_traces::CallKey, + #[case] call_id: &str, + #[case] response_id: Option<&str>, + #[case] expected: Option, +) { + let keys = [ + transport, + litellm_traces::CallKey::LiteLlmRequest(call_id.into()), + ] + .into_iter() + .chain(response_id.map(|id| litellm_traces::CallKey::ProviderResponse(id.into()))) + .collect(); + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(llm("call", "agent", "agent", ""), "team", "", "key"), + owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: keys, + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..row("attempt", "call", "gateway.request", "framework", "") + }, + "team", + "", + "key", + ), + ]; + let logs = [ + SpendByResponseIdsRow { + litellm_call_id: "call-a".into(), + trace_id: "trace".into(), + span_id: "attempt".into(), + ..owned_spend("request-a", "response-a", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + litellm_call_id: "call-b".into(), + trace_id: "trace".into(), + span_id: "other-attempt".into(), + ..owned_spend("request-b", "response-b", "team", "", "key", 0.5) + }, + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); + assert_eq!(trace.spans[2].spend, expected); +} + +#[rstest] +#[case::legacy_row("", Some(0.5))] +#[case::other_call("other-call", None)] +fn gateway_id_miss_only_vetoes_rows_that_carry_a_call_id( + #[case] logged_call_id: &str, + #[case] expected: Option, +) { + let mut transport = row("http", "llm", "gateway.request", "framework", ""); + transport.trace_id = "trace".into(); + transport.call_keys = vec![ + "transport:".parse().unwrap(), + "litellm_request:gateway-call".parse().unwrap(), + ]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + let mut call = llm("llm", "agent", "agent", ""); + call.trace_id = "trace".into(); + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(call, "team", "", "key"), + owned(transport, "team", "", "key"), + ]; + let mut logged = owned_spend("chatcmpl-1", "chatcmpl-1", "team", "", "key", 0.5); + logged.trace_id = "trace".into(); + logged.span_id = "http".into(); + logged.litellm_call_id = logged_call_id.into(); + let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); + assert_eq!(trace.summary.spend, expected); +} + +#[rstest] +#[case::narrows_ambiguity("request-a", Some(0.25))] +#[case::conflicting_exact_request("request-c", None)] +fn complete_wrapper_reconciles_ambiguous_response( + #[case] exact_id: &str, + #[case] expected: Option, +) { + let wrapper = TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest(exact_id.to_owned())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..owned(llm("wrapper", "", "agent", ""), "team", "", "key") + }; + let rows = [ + wrapper, + owned( + llm("call", "wrapper", "agent", "response"), + "team", + "", + "key", + ), + ]; + let logs = [ + owned_spend("request-a", "response", "team", "", "key", 0.25), + owned_spend("request-b", "response", "team", "", "key", 0.5), + owned_spend("request-c", "other-response", "team", "", "key", 0.75), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::complete_retry(true, litellm_traces::CallEvidenceKind::Complete, Some(0.75))] +#[case::missing_retry(false, litellm_traces::CallEvidenceKind::Complete, None)] +#[case::unknown_retry(true, litellm_traces::CallEvidenceKind::Unknown, None)] +#[case::partial_retry(true, litellm_traces::CallEvidenceKind::Partial, None)] +fn transports_preserve_retry_spend_without_counting_unrelated_cached_rows( + #[case] retry_logged: bool, + #[case] retry_evidence: litellm_traces::CallEvidenceKind, + #[case] expected: Option, +) { + let transport = |id: &str| { + owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec!["transport:".parse().unwrap()], + call_evidence: Some(if id == "first" { + retry_evidence + } else { + litellm_traces::CallEvidenceKind::Complete + }), + ..row(id, "call", "POST", "framework", "") + }, + "team", + "", + "key", + ) + }; + let rows = [ + owned( + llm("call", "", "agent", "final-response"), + "team", + "", + "key", + ), + transport("first"), + transport("second"), + ]; + let logs = [ + SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "first".into(), + ..owned_spend("retry", "retry-response", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "second".into(), + ..owned_spend("final", "final-response", "team", "", "key", 0.5) + }, + owned_spend("cached", "final-response", "team", "", "key", 0.0), + ]; + let available = if retry_logged { &logs[..] } else { &logs[1..] }; + let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::same_request(false)] +#[case::ambiguous_response(true)] +fn multiple_identifiers_for_one_request_count_its_spend_once(#[case] cached_row: bool) { + let rows = [owned( + TraceSpansRow { + call_keys: vec![ + "provider_response:response".parse().unwrap(), + "litellm_request:request".parse().unwrap(), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "response") + }, + "team", + "", + "key", + )]; + let logs = [ + owned_spend("request", "response", "team", "", "key", 0.25), + owned_spend("cached", "response", "team", "", "key", 0.5), + ]; + let available = if cached_row { &logs[..] } else { &logs[..1] }; + let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!(trace.agents[0].spend, Some(0.25)); + assert_eq!(trace.spans[0].spend, Some(0.25)); +} + +#[rstest] +fn complete_wrapper_accounts_for_retries_missing_from_the_call_span() { + let rows = [ + owned( + TraceSpansRow { + call_keys: vec![ + "litellm_request:retry".parse().unwrap(), + "litellm_request:final".parse().unwrap(), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("wrapper", "", "agent", "") + }, + "team", + "", + "key", + ), + owned( + llm("call", "wrapper", "agent", "response"), + "team", + "", + "key", + ), + ]; + let logs = [ + owned_spend("retry", "retry-response", "team", "", "key", 0.25), + owned_spend("final", "response", "team", "", "key", 0.5), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.75)); + assert_eq!(trace.agents[0].spend, Some(0.75)); +} + +#[rstest] +#[case::legacy(None, Some(0.25))] +#[case::unknown(Some(litellm_traces::CallEvidenceKind::Unknown), None)] +#[case::partial(Some(litellm_traces::CallEvidenceKind::Partial), None)] +#[case::complete(Some(litellm_traces::CallEvidenceKind::Complete), Some(0.25))] +fn legacy_request_id_fallback_respects_recorded_evidence( + #[case] evidence: Option, + #[case] expected: Option, +) { + let span = owned( + TraceSpansRow { + call_evidence: evidence, + ..llm("call", "", "agent", "response") + }, + "team", + "", + "key", + ); + let stored = serde_json::to_value(span).unwrap(); + let decoded: TraceSpansRow = serde_json::from_value(stored).unwrap(); + let logs = [owned_spend("request", "response", "team", "", "key", 0.25)]; + let trace = resolve_trace("trace", "ref", &[decoded], &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.spans[0].spend, expected); +} + +#[rstest] +#[case::unknown(litellm_traces::CallEvidenceKind::Unknown)] +#[case::complete(litellm_traces::CallEvidenceKind::Complete)] +fn spend_lookup_fetches_recorded_keys_before_resolving_completeness( + #[case] evidence: litellm_traces::CallEvidenceKind, +) { + let recorded = TraceSpansRow { + trace_id: "trace".to_owned(), + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("response".to_owned()), + litellm_traces::CallKey::LiteLlmRequest("request".to_owned()), + litellm_traces::CallKey::Transport, + ], + call_evidence: Some(evidence), + ..row("span", "", "operation", "llm", "") + }; + let lookup = litellm_traces::SpendLookup::new(&[recorded]); + assert_eq!(lookup.response_ids, ["response"]); + assert_eq!(lookup.request_ids, ["request"]); + assert_eq!(lookup.trace_ids, ["trace"]); +} + +#[rstest] +#[case::gateway(false)] +#[case::transport(true)] +fn independent_key_disambiguates_repeated_request_ids(#[case] transport: bool) { + let rows = [owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("response".into()), + if transport { + litellm_traces::CallKey::Transport + } else { + litellm_traces::CallKey::LiteLlmRequest("gateway".into()) + }, + ], + ..llm("call", "", "agent", "response") + }, + "team", + "", + "key", + )]; + let logs = [ + SpendByResponseIdsRow { + start_ms: 100, + litellm_call_id: "gateway".into(), + trace_id: "trace".into(), + span_id: "call".into(), + ..owned_spend("same", "response", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + start_ms: 200, + litellm_call_id: "other".into(), + ..owned_spend("same", "response", "team", "", "key", 0.5) + }, + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!(trace.spans[0].spend, Some(0.25)); +} + +#[rstest] +fn conflicting_keys_cannot_agree_on_request_id_alone() { + let rows = [ + owned( + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + ..llm("wrapper", "", "agent", "") + }, + "team", + "", + "key", + ), + owned( + llm("call", "wrapper", "agent", "response"), + "team", + "", + "key", + ), + ]; + let logs = [ + SpendByResponseIdsRow { + start_ms: 100, + ..owned_spend("same", "response", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + start_ms: 200, + litellm_call_id: "gateway".into(), + ..owned_spend("same", "other", "team", "", "key", 0.5) + }, + ]; + assert_eq!( + resolve_trace("trace", "ref", &rows, &logs) + .unwrap() + .summary + .spend, + None + ); +} + +#[rstest] +#[case::gateway("gateway", "provider-id", "team", "key", Some(0.25))] +#[case::legacy("", "gateway", "team", "key", Some(0.25))] +#[case::conflict("other", "gateway", "team", "key", None)] +#[case::other_team("gateway", "provider-id", "other-team", "key", None)] +#[case::other_key("gateway", "provider-id", "team", "other-key", None)] +fn gateway_lookup_respects_legacy_fallback_and_ownership( + #[case] call_id: &str, + #[case] request_id: &str, + #[case] team: &str, + #[case] key: &str, + #[case] expected: Option, +) { + let rows = [owned( + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + ..llm("call", "", "agent", "") + }, + "team", + "", + "key", + )]; + let logs = [SpendByResponseIdsRow { + litellm_call_id: call_id.into(), + ..owned_spend(request_id, "provider", team, "", key, 0.25) + }]; + assert_eq!( + resolve_trace("trace", "ref", &rows, &logs) + .unwrap() + .summary + .spend, + expected + ); +} + +#[rstest] +#[case::native_id("provider_request:req_native", false, true)] +#[case::historical_id("provider_response:req_native", false, true)] +#[case::legacy_message_id("provider_response:msg_native", false, true)] +#[case::transport("", true, true)] +#[case::no_correlation("", false, false)] +fn native_cost_uses_typed_ids_or_the_original_transport( + #[case] encoded: &str, + #[case] traceparent: bool, + #[case] priced: bool, + #[values(false, true)] grouped: bool, +) { + let call = TraceSpansRow { + trace_id: if grouped { + "session-trace" + } else { + "original-trace" + } + .into(), + original_trace_id: if grouped { "original-trace" } else { "" }.into(), + name: "claude_code.llm_request".into(), + framework: "claude-code".into(), + call_keys: encoded.parse().into_iter().collect(), + call_evidence: Some(litellm_traces::CallEvidenceKind::Unknown), + ..llm("native-call", "", "agent", "") + }; + let log = SpendByResponseIdsRow { + provider_request_id: "req_native".into(), + trace_id: if traceparent { "original-trace" } else { "" }.into(), + span_id: if traceparent { "native-call" } else { "" }.into(), + ..spend("gateway-log", "msg_native", 0.25) + }; + let lookup = litellm_traces::SpendLookup::new(std::slice::from_ref(&call)); + let trace = resolve_trace("trace", "ref", &[call], &[log]).unwrap(); + assert_eq!( + (trace.summary.spend, trace.summary.priced_calls), + (priced.then_some(0.25), u64::from(priced)) + ); + assert_eq!(trace.spans[0].spend, priced.then_some(0.25)); + if encoded.is_empty() { + assert_eq!(lookup.trace_ids, ["original-trace"]); + } else if !encoded.ends_with("msg_native") { + assert_eq!(lookup.provider_request_ids, ["req_native"]); + assert!(lookup.response_ids.is_empty()); + } +} + +#[rstest] +fn provider_request_id_cannot_match_a_message_id_of_the_same_value() { + let call = TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::ProviderRequest( + "req_native".into(), + )], + ..llm("call", "", "agent", "") + }; + let rows = [spend("unrelated", "req_native", 0.25)]; + let trace = resolve_trace("trace", "ref", &[call], &rows).unwrap(); + assert_eq!(trace.summary.spend, None); + assert_eq!(trace.spans[0].spend_match, Some(SpendMatch::NoSpendLog)); +} diff --git a/litellm/integrations/clickhouse/clickhouse_spend_logger.py b/litellm/integrations/clickhouse/clickhouse_spend_logger.py index 588f81ec50a..6c9f726056b 100644 --- a/litellm/integrations/clickhouse/clickhouse_spend_logger.py +++ b/litellm/integrations/clickhouse/clickhouse_spend_logger.py @@ -19,6 +19,7 @@ from litellm._logging import verbose_logger from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger from litellm.integrations.clickhouse.context import is_lens_analysis from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE +from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.tracing.types import SpendLogRecord from litellm.types.utils import StandardLoggingPayload @@ -155,6 +156,11 @@ def spend_log_row_from_payload(payload: StandardLoggingPayload, kwargs: Mapping[ return SpendLogRecord( request_id=request_id, response_id=strip_cache_hit_suffix(request_id), + provider_request_id=( + get_provider_request_id(kwargs.get("response_headers")) + or get_provider_request_id(hidden_params.get("additional_headers")) + or "" + ), litellm_call_id=payload.get("litellm_call_id") or "", call_type=payload.get("call_type") or "", api_key=metadata.get("user_api_key_hash") or "", diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 4fa78640e24..76ea2c25159 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -11,6 +11,7 @@ import httpx from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason if TYPE_CHECKING: @@ -495,7 +496,8 @@ def process_response_headers( **processed_headers, **additional_headers, } - return additional_headers + request_id: Final = get_provider_request_id(response_headers) + return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})} def preserve_upstream_non_openai_attributes( diff --git a/litellm/litellm_core_utils/llm_response_utils/get_headers.py b/litellm/litellm_core_utils/llm_response_utils/get_headers.py index d04abcb6e7b..b86cc954b0e 100644 --- a/litellm/litellm_core_utils/llm_response_utils/get_headers.py +++ b/litellm/litellm_core_utils/llm_response_utils/get_headers.py @@ -1,6 +1,20 @@ from collections.abc import Mapping from typing import Final +from pydantic import TypeAdapter, ValidationError + +_HEADERS: Final = TypeAdapter(Mapping[str, str]) +_REQUEST_ID_HEADERS: Final = ("request-id", "x-request-id", "llm_provider-request-id", "llm_provider-x-request-id") + + +def get_provider_request_id(headers: object) -> str | None: + try: + values: Final = _HEADERS.validate_python(headers) + except ValidationError: + return None + normalized: Final = {name.lower(): value for name, value in values.items()} + return next((normalized[name] for name in _REQUEST_ID_HEADERS if normalized.get(name)), None) + def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> dict: """ @@ -29,7 +43,9 @@ def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> if "x-ratelimit-remaining-tokens" in _response_headers: openai_headers["x-ratelimit-remaining-tokens"] = _response_headers["x-ratelimit-remaining-tokens"] llm_provider_headers: Final = _get_llm_provider_headers(_response_headers) - return {**llm_provider_headers, **openai_headers} + request_id: Final = get_provider_request_id(_response_headers) + native_headers: Final = {"request-id": request_id} if request_id is not None else {} + return {**llm_provider_headers, **openai_headers, **native_headers} def _get_llm_provider_headers(response_headers: Mapping[str, str]) -> dict: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 86c36a7b413..53ea85a2740 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2214,6 +2214,7 @@ class BaseLLMHTTPHandler: # used for logging + cost tracking logging_obj.model_call_details["httpx_response"] = response + logging_obj.model_call_details["response_headers"] = dict(response.headers) initial_response: AsyncIterator | AnthropicMessagesResponse if stream: diff --git a/litellm/rust_bridge/callbacks_legacy_python.py b/litellm/rust_bridge/callbacks_legacy_python.py index e011b795000..46af4e236eb 100644 --- a/litellm/rust_bridge/callbacks_legacy_python.py +++ b/litellm/rust_bridge/callbacks_legacy_python.py @@ -21,6 +21,8 @@ from typing import ( cast, # noqa: TID251 # bounded compatibility calls into legacy Python integrations ) +from pydantic import TypeAdapter + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging @@ -337,9 +339,15 @@ def after_deployment_failure(kwargs: dict[str, object], error: Exception, call_t return hook(kwargs, error, call_type) -def stream_opened(logger: Logging) -> None: +_STREAM_HEADERS: Final = TypeAdapter(Mapping[str, str]) + + +def stream_opened(logger: Logging, head: Mapping[str, object]) -> None: logger.stream = True logger.model_call_details["stream"] = True + logger.model_call_details["response_headers"] = _STREAM_HEADERS.validate_python( + head.get("additional_headers") or {} + ) def stream_success( diff --git a/litellm/rust_bridge/trace/generated/types.py b/litellm/rust_bridge/trace/generated/types.py index 127e86e9160..4a09e731228 100644 --- a/litellm/rust_bridge/trace/generated/types.py +++ b/litellm/rust_bridge/trace/generated/types.py @@ -59,6 +59,7 @@ class AgentNode(typing_extensions.TypedDict): tool_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] duration_ms: ReadOnly[float] spend: ReadOnly[float | None] + priced_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] SpanType: TypeAlias = Literal[ @@ -77,6 +78,9 @@ SpanType: TypeAlias = Literal[ ] +SpendMatch: TypeAlias = Literal["matched", "no_call_id", "no_spend_log", "ambiguous", "incomplete_evidence"] + + class TraceScope(typing_extensions.TypedDict): all_teams: ReadOnly[Literal[0, 1]] user_id: ReadOnly[str] @@ -120,6 +124,7 @@ class TraceSummary(typing_extensions.TypedDict): output_tokens: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] models: ReadOnly[tuple[str, ...]] spend: ReadOnly[float | None] + priced_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] class Span(typing_extensions.TypedDict): @@ -140,6 +145,8 @@ class Span(typing_extensions.TypedDict): output_tokens: ReadOnly[Annotated[int, Field(ge=0, le=4294967295)]] litellm_request_id: ReadOnly[str | None] spend: ReadOnly[float | None] + spend_log_request_id: ReadOnly[str | None] + spend_match: ReadOnly[SpendMatch | None | None] class Trace(typing_extensions.TypedDict): diff --git a/litellm/tracing/types.py b/litellm/tracing/types.py index e9930dc7bf7..8bd119940c2 100644 --- a/litellm/tracing/types.py +++ b/litellm/tracing/types.py @@ -8,6 +8,7 @@ class SpendLogRecord(TypedDict): request_id: ReadOnly[str] response_id: ReadOnly[str] + provider_request_id: ReadOnly[str] litellm_call_id: ReadOnly[str] call_type: ReadOnly[str] api_key: ReadOnly[str] diff --git a/scripts/trace_codegen/schemas/traces/Trace.json b/scripts/trace_codegen/schemas/traces/Trace.json index dc66f77412c..ebd99880956 100644 --- a/scripts/trace_codegen/schemas/traces/Trace.json +++ b/scripts/trace_codegen/schemas/traces/Trace.json @@ -28,6 +28,12 @@ "null" ] }, + "priced_calls": { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, "spend": { "format": "double", "type": [ @@ -49,7 +55,8 @@ "llm_calls", "tool_calls", "duration_ms", - "spend" + "spend", + "priced_calls" ], "type": "object" }, @@ -120,6 +127,22 @@ "null" ] }, + "spend_log_request_id": { + "type": [ + "string", + "null" + ] + }, + "spend_match": { + "anyOf": [ + { + "$ref": "#/$defs/SpendMatch" + }, + { + "type": "null" + } + ] + }, "start_offset_ms": { "format": "double", "type": "number" @@ -148,7 +171,9 @@ "input_tokens", "output_tokens", "litellm_request_id", - "spend" + "spend", + "spend_log_request_id", + "spend_match" ], "type": "object" }, @@ -177,6 +202,16 @@ ], "type": "string" }, + "SpendMatch": { + "enum": [ + "matched", + "no_call_id", + "no_spend_log", + "ambiguous", + "incomplete_evidence" + ], + "type": "string" + }, "TraceSummary": { "properties": { "agent_count": { @@ -245,6 +280,12 @@ "minimum": 0, "type": "integer" }, + "priced_calls": { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, "resolution_limited": { "type": "boolean", "x-python-optional": true @@ -306,7 +347,8 @@ "input_tokens", "output_tokens", "models", - "spend" + "spend", + "priced_calls" ], "type": "object" } diff --git a/scripts/trace_codegen/schemas/traces/TracePage.json b/scripts/trace_codegen/schemas/traces/TracePage.json index dfa02ba0d39..429635ec3b1 100644 --- a/scripts/trace_codegen/schemas/traces/TracePage.json +++ b/scripts/trace_codegen/schemas/traces/TracePage.json @@ -76,6 +76,12 @@ "minimum": 0, "type": "integer" }, + "priced_calls": { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, "resolution_limited": { "type": "boolean", "x-python-optional": true @@ -137,7 +143,8 @@ "input_tokens", "output_tokens", "models", - "spend" + "spend", + "priced_calls" ], "type": "object" } diff --git a/tests/llm_translation/test_llm_response_utils/test_get_headers.py b/tests/llm_translation/test_llm_response_utils/test_get_headers.py index 380f89bbdd4..60a6a4bebcd 100644 --- a/tests/llm_translation/test_llm_response_utils/test_get_headers.py +++ b/tests/llm_translation/test_llm_response_utils/test_get_headers.py @@ -8,6 +8,7 @@ import pytest from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, _get_llm_provider_headers, + get_provider_request_id, ) @@ -72,3 +73,17 @@ def test_get_llm_provider_headers(): } result = _get_llm_provider_headers(input_headers) assert result == expected_output, "Unexpected output for _get_llm_provider_headers" + + +@pytest.mark.parametrize("header", ("request-id", "Request-Id", "x-request-id", "llm_provider-request-id")) +def test_native_clients_receive_the_provider_request_id(header: str) -> None: + result = get_response_headers({header: "req_test", "unrelated": "value"}) + assert result["request-id"] == "req_test" + assert get_provider_request_id(result) == "req_test" + assert result["llm_provider-unrelated"] == "value" + assert "unrelated" not in result + + +@pytest.mark.parametrize("headers", (None, {}, {"request-id": ""}, {"request-id": 42})) +def test_invalid_provider_request_ids_remain_absent(headers: object) -> None: + assert get_provider_request_id(headers) is None diff --git a/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py b/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py index 491f8651b52..fe47c1a59f3 100644 --- a/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py +++ b/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py @@ -163,6 +163,7 @@ def test_success_row_mapping(): assert row["request_id"] == "chatcmpl-abc123" assert row["response_id"] == "chatcmpl-abc123" assert row["litellm_call_id"] == "gateway-call" + assert row["provider_request_id"] == "" assert row["spend"] == 0.00042 assert (row["prompt_tokens"], row["completion_tokens"], row["total_tokens"]) == (20, 10, 30) assert (row["cache_read_tokens"], row["cache_write_tokens"]) == (5, 7) @@ -186,6 +187,18 @@ def test_success_row_mapping(): assert json.loads(row["metadata"])["user_api_key_alias"] == "my-key" +@pytest.mark.parametrize("header_source", ("response_headers", "additional_headers")) +def test_provider_request_id_stays_separate_from_message_and_gateway_ids(header_source: str) -> None: + headers: Final = {"request-id": "req_native"} + hidden: Final = {"additional_headers": headers} if header_source == "additional_headers" else {} + kwargs: Final = {"response_cost": 0.00042, "response_headers": headers if header_source == "response_headers" else None} + payload: Final = cast(StandardLoggingPayload, _payload(id="msg_native", hidden_params=hidden)) + row: Final = spend_log_row_from_payload(payload, kwargs) + assert (row["provider_request_id"], row["response_id"], row["litellm_call_id"]) == ( + "req_native", "msg_native", "gateway-call" + ) + + @pytest.mark.parametrize("status", ("success", "failure")) @pytest.mark.asyncio async def test_custom_request_metadata_is_redacted_before_clickhouse_logging( diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 2a6dd347d5f..9f48ea62a72 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -1,6 +1,7 @@ """Tests for litellm_core_utils.core_helpers module.""" import logging +from typing import Final import httpx import pytest @@ -16,6 +17,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_provider_response_headers_from_hidden_params, map_finish_reason, normalize_drop_params, + process_response_headers, reconstruct_model_name, redact_nested_match_and_regex_keys, set_provider_response_headers_in_hidden_params, @@ -25,6 +27,14 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ImageResponse, TranscriptionResponse +@pytest.mark.parametrize("header", ("request-id", "x-request-id", "llm_provider-request-id")) +def test_native_request_id_survives_stream_header_processing(header: str) -> None: + processed: Final = process_response_headers(httpx.Headers({header: "req_native"})) + + assert processed["request-id"] == "req_native" + assert processed[header if header.startswith("llm_provider-") else "llm_provider-" + header] == "req_native" + + class TestBudgetReservationBinding: """The request-end release skips a reservation a cost callback has claimed, so the claim must land on the one dict auth stamped, through whichever metadata field or auth object diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 93d18727f22..2d59ab337d4 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -895,6 +895,7 @@ async def test_async_anthropic_messages_handler_streaming_forwards_provider_resp additional_headers = result._hidden_params["additional_headers"] assert additional_headers["llm_provider-x-amzn-requestid"] == "amzn-req-123" assert additional_headers["llm_provider-x-amzn-trace-id"] == "Root=1-abc-def" + assert mock_logging_obj.model_call_details["response_headers"] == dict(upstream_response.headers) collected = b"".join([chunk async for chunk in result]) assert b"message_start" in collected @@ -1601,8 +1602,7 @@ async def test_async_anthropic_messages_handler_forwards_request_timeout(monkeyp expected_response = {"id": "msg_1", "content": []} mock_config.transform_anthropic_messages_response = Mock(return_value=expected_response) - ok_response = Mock() - ok_response.raise_for_status = Mock(return_value=None) + ok_response = httpx.Response(200, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")) mock_client = AsyncMock(spec=AsyncHTTPHandler) mock_client.post = AsyncMock(return_value=ok_response) @@ -2072,8 +2072,7 @@ async def test_async_anthropic_messages_handler_passes_api_key_to_agentic_hooks( captured_kwargs.update(call_kwargs) return sentinel_response - mock_httpx_response = Mock() - mock_httpx_response.status_code = 200 + mock_httpx_response = httpx.Response(200, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")) with ( patch.object( diff --git a/tests/unit/rust_bridge/messages/test_route_host.py b/tests/unit/rust_bridge/messages/test_route_host.py index 1be42e2249d..7b15553a055 100644 --- a/tests/unit/rust_bridge/messages/test_route_host.py +++ b/tests/unit/rust_bridge/messages/test_route_host.py @@ -161,4 +161,4 @@ def test_stream_hidden_params_projects_upstream_headers_the_way_the_python_handl assert isinstance(additional, dict) assert additional["llm_provider-request-id"] == "req_upstream_123" assert additional["x-ratelimit-remaining-requests"] == "41" - assert "request-id" not in additional + assert additional["request-id"] == additional["llm_provider-request-id"] diff --git a/tests/unit/rust_bridge/test_callbacks_legacy_python.py b/tests/unit/rust_bridge/test_callbacks_legacy_python.py index 045dcb52ce9..c78fbb3f246 100644 --- a/tests/unit/rust_bridge/test_callbacks_legacy_python.py +++ b/tests/unit/rust_bridge/test_callbacks_legacy_python.py @@ -34,6 +34,14 @@ def _supplied_logger() -> Logging: ) +def test_native_stream_headers_reach_spend_callbacks() -> None: + logger: Final = _supplied_logger() + legacy.stream_opened(logger, {"additional_headers": {"llm_provider-request-id": "req_native"}}) + + assert logger.stream is True + assert logger.model_call_details["response_headers"] == {"llm_provider-request-id": "req_native"} + + def test_setup_reuses_a_supplied_logger() -> None: supplied: Final = _supplied_logger() result: Final = setup( @@ -159,9 +167,7 @@ def test_failure_handler_of_an_internal_call_leaves_the_outer_budget_reservation pending.close() -CONTRACT_PATH: Final = ( - Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" -) +CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" def test_the_rust_contract_matches_the_shim_signatures() -> None: diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts b/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts index 27c46a528e7..f33f86cd280 100644 --- a/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts +++ b/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts @@ -26,6 +26,8 @@ function makeTrace(scene: Scenario, index: number, now: number) { parent_span_id: null, span_id: spanId(0), spend: null, + spend_log_request_id: null, + spend_match: null, start_offset_ms: 0, status: "ok", type: "agent", @@ -53,6 +55,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { input_tokens: 520 + index * 41, output_tokens: 48 + index * 7, spend: 0.003 + index * 0.0002, + spend_match: "matched", }; const trace: Trace = { summary: { @@ -71,6 +74,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { service: "demo-agents", span_count: toolCount + 2, spend: model.spend, + priced_calls: 1, start_time: iso(now - (index + 1) * 35 * 60_000), status: scene.failed ? "error" : "ok", tool_calls: toolCount, @@ -85,6 +89,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { llm_calls: 1, tool_calls: toolCount, spend: model.spend, + priced_calls: 1, }, ], spans: [base, ...tools, model], diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts b/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts index 866f1042b73..44eac42755c 100644 --- a/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts +++ b/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts @@ -85,6 +85,7 @@ export function withReleaseCases(run: { trace: Trace; details: SpanDetail[] }) { input_tokens: models.reduce((sum, span) => sum + span.input_tokens, 0), output_tokens: models.reduce((sum, span) => sum + span.output_tokens, 0), spend: models.reduce((sum, span) => sum + (span.spend ?? 0), 0), + priced_calls: models.length, }; const finalDetail = run.details.at(-1)!; const history = caseDetails.filter((_, index) => index % 3 === 2); @@ -102,6 +103,7 @@ export function withReleaseCases(run: { trace: Trace; details: SpanDetail[] }) { llm_calls: caseCount, tool_calls: caseCount, spend: summary.spend - (final.spend ?? 0), + priced_calls: caseCount, }, ], }, diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx index 16afa455be6..33fb073d0a9 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx @@ -10,8 +10,8 @@ import { cn } from "@/lib/cva.config"; import { copyToClipboard } from "@/utils/dataUtils"; import type { TraceHandoff } from "../../api"; +import { runCost } from "../../list/AgentTracesTable"; import { traceRefOf, traceShareUrl } from "../../routing"; -import { formatCost } from "../../list/AgentTracesTable"; import { IdChip } from "../../ui/IdChip"; import { SpanIcon } from "../../ui/SpanIcon"; import { FrameworkLogo, traceFramework } from "../../ui/TraceFramework"; @@ -89,6 +89,7 @@ interface RunHeaderProps { export function RunHeader({ trace, handoff, onBack, embedded }: RunHeaderProps) { const { summary } = trace; const failed = summary.status === "error"; + const cost = runCost(summary); return (
@@ -127,7 +128,10 @@ export function RunHeader({ trace, handoff, onBack, embedded }: RunHeaderProps) - + {summary.error_count > 0 && }
diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx index fb9285d0777..cf2c32ff5c7 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx @@ -13,10 +13,31 @@ import type { Span, SpanDetail, SpanErrorPage, Trace } from "../../types"; vi.mock("../../../../networking", () => ({ agentTraceSpanCall: vi.fn(), agentTraceSpanErrorCall: vi.fn(), + uiSpendLogsCall: vi.fn(), getProxyBaseUrl: () => "http://proxy.test/", })); -import { agentTraceSpanCall, agentTraceSpanErrorCall } from "../../../../networking"; +vi.mock("../../../../logs/detail", () => ({ + LogDetailsDrawer: ({ + open, + logEntry, + onClose, + backTo, + }: { + open: boolean; + logEntry: { request_id: string } | null; + onClose: () => void; + backTo?: { label: string }; + }) => + open && logEntry ? ( +
+ {logEntry.request_id} + {backTo && } +
+ ) : null, +})); + +import { agentTraceSpanCall, agentTraceSpanErrorCall, uiSpendLogsCall } from "../../../../networking"; type SpanFields = Partial & Pick; @@ -37,6 +58,8 @@ const span = (overrides: SpanFields): Span => ({ output_tokens: 0, litellm_request_id: null, spend: null, + spend_log_request_id: null, + spend_match: null, ...overrides, }); @@ -49,6 +72,9 @@ const llmFields: SpanFields = { input_tokens: 659, output_tokens: 60, litellm_request_id: "chatcmpl-abc", + spend: 0.0002, + spend_log_request_id: "req_7f3a9c2e1b44", + spend_match: "matched", }; const failedToolFields: SpanFields = { span_id: "tool1", @@ -244,13 +270,47 @@ describe("DetailPane", () => { expect(input).toHaveTextContent("acme-404"); }); - it("shows the LiteLLM request facts on the Request tab", async () => { + it("links a priced LLM step to its spend log in the header and opens it by that request id", async () => { const user = userEvent.setup(); + vi.mocked(uiSpendLogsCall).mockResolvedValue({ data: [{ request_id: "req_7f3a9c2e1b44" }] }); renderPane(spanRow(llm)); + const link = screen.getByRole("button", { name: "Open LiteLLM spend log req_7f3a9c2e1b44" }); + expect(link).toHaveTextContent("LiteLLM Spend Log$0.0002"); + await user.click(link); + expect(await screen.findByRole("dialog", { name: "Request log" })).toHaveTextContent("req_7f3a9c2e1b44"); + expect(vi.mocked(uiSpendLogsCall).mock.calls[0][0].params).toEqual({ request_id: "req_7f3a9c2e1b44" }); + }); + + it("returns from the spend log to the same agent trace step", async () => { + const user = userEvent.setup(); + vi.mocked(uiSpendLogsCall).mockResolvedValue({ data: [{ request_id: "req_7f3a9c2e1b44" }] }); + renderPane(spanRow(llm)); + await user.click(screen.getByRole("button", { name: "Open LiteLLM spend log req_7f3a9c2e1b44" })); + const drawer = await screen.findByRole("dialog", { name: "Request log" }); + await user.click(within(drawer).getByRole("button", { name: "Back to Lens trace" })); + await waitFor(() => expect(screen.queryByRole("dialog", { name: "Request log" })).not.toBeInTheDocument()); + expect(screen.getByRole("button", { name: "Open LiteLLM spend log req_7f3a9c2e1b44" })).toBeVisible(); + }); + + it.each([ + { spend_match: "no_call_id" as const, reason: "This step records no usable call identifier or transport context" }, + { spend_match: "no_spend_log" as const, reason: "No matching spend log belongs to this run" }, + { spend_match: "incomplete_evidence" as const, reason: "This step does not account for every request in the call" }, + ])("says why an unpriced LLM step has no cost ($spend_match)", async ({ spend_match, reason }) => { + const user = userEvent.setup(); + const unpriced: SpanFields = { ...llmFields, spend: null, spend_log_request_id: null, spend_match }; + renderPane(spanRow(span(unpriced))); + expect(screen.getByTitle(reason)).toHaveTextContent("Cost not matched"); + expect(screen.queryByRole("button", { name: /Open LiteLLM spend log/ })).not.toBeInTheDocument(); await user.click(screen.getByRole("tab", { name: "Request" })); - expect(await screen.findByText("chatcmpl-abc")).toBeInTheDocument(); - expect(screen.getByText("659")).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /Open request log/ })).toBeInTheDocument(); + expect(await screen.findByText(reason, { selector: "p" })).toBeInTheDocument(); + }); + + it("keeps the Cost fact for a priced step that has no single spend log to link", () => { + const pricedFromRetries: SpanFields = { ...llmFields, spend: 0.0005, spend_log_request_id: null }; + renderPane(spanRow(span(pricedFromRetries))); + expect(screen.queryByRole("button", { name: /Open LiteLLM spend log/ })).not.toBeInTheDocument(); + expect(screen.getByText("Cost").parentElement).toHaveTextContent("Cost$0.0005"); }); it("summarizes a ×N group with its failure pattern", () => { diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx index 4757c55ab80..240ea149199 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx @@ -16,11 +16,12 @@ interface PaneHeaderProps { idValue?: string; facts: readonly (readonly [label: string, value: string])[]; actions?: React.ReactNode; + links?: React.ReactNode; onClose: () => void; } /** Selected step identity: tile, name, id, then its time and usage on one quiet line. */ -export function PaneHeader({ type, model, failed, title, idValue, facts, actions, onClose }: PaneHeaderProps) { +export function PaneHeader({ type, model, failed, title, idValue, facts, actions, links, onClose }: PaneHeaderProps) { return (
@@ -40,15 +41,18 @@ export function PaneHeader({ type, model, failed, title, idValue, facts, actions
- {facts.length > 0 && ( -
- {facts.map(([label, value]) => ( -
-
{label}
-
{value}
-
- ))} -
+ {(facts.length > 0 || links) && ( +
+
+ {facts.map(([label, value]) => ( +
+
{label}
+
{value}
+
+ ))} +
+ {links} +
)} ); diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx index 855a7060336..cd36d2686ef 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx @@ -1,19 +1,14 @@ "use client"; -import { ArrowUpRight } from "lucide-react"; -import { useState } from "react"; - import CopyButton from "@/components/shared/CopyButton"; -import { Button } from "@/components/ui/button"; -import { LogDetailsDrawer } from "../../../../logs/detail"; import { fieldEntries } from "../content/payload"; import { formatCost } from "../../list/AgentTracesTable"; import type { Span } from "../../types"; import { fmtMs, fmtTok } from "../../utils"; -import { useSpanRequestLog } from "../useSpanRequestLog"; import { FieldTree } from "../content/FieldTree"; import { DetailGroup } from "./DetailGroup"; +import { SpendLogLink, unmatchedReason } from "./SpendLogLink"; interface RequestTabProps { span: Span; @@ -22,17 +17,7 @@ interface RequestTabProps { traceStartMs: number; } -/** Usage and the linked LiteLLM request for an LLM span; "Open request log" opens the request drawer over the run. */ export function RequestTab({ span, accessToken, traceStartMs }: RequestTabProps) { - const [drawerOpen, setDrawerOpen] = useState(false); - const logQuery = useSpanRequestLog( - accessToken, - span.litellm_request_id, - traceStartMs + span.start_offset_ms, - drawerOpen, - ); - const logNotFound = drawerOpen && logQuery.isSuccess && logQuery.data === null; - if (span.type !== "llm") { return
This span is not a model request.
; } @@ -51,30 +36,22 @@ export function RequestTab({ span, accessToken, traceStartMs }: RequestTabProps) - - {span.litellm_request_id ? ( -
+ +
+ {span.spend_log_request_id && (
- +
- +
- - {logNotFound &&

No request log found for this call.

} + )} +
+
- ) : ( -

Not linked to a LiteLLM request

- )} + {unmatchedReason(span) &&

{unmatchedReason(span)}

} +
- setDrawerOpen(false)} - logEntry={logQuery.data ?? null} - accessToken={accessToken} - />
); } diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx index cfa92f67577..d74d7ad8ccd 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx @@ -12,6 +12,7 @@ import { ContentTab, useSpanDetail } from "../content/ContentTab"; import { AttributesTab } from "./AttributesTab"; import { PaneHeader } from "./PaneHeader"; import { RequestTab } from "./RequestTab"; +import { SpendLogLink } from "./SpendLogLink"; const TAB_LABELS: Record = { content: "Content", request: "Request", attributes: "Attributes" }; @@ -26,7 +27,7 @@ const spanFacts = (span: Span): readonly Fact[] => { const tokens = span.input_tokens + span.output_tokens; const optional: readonly (Fact | null)[] = [ tokens > 0 ? ["Tokens", fmtTok(tokens)] : null, - span.spend != null ? ["Cost", formatCost(span.spend)] : null, + span.spend != null && span.spend_log_request_id == null ? ["Cost", formatCost(span.spend)] : null, span.type === "llm" && span.model ? ["Step", span.name] : null, ]; return [["Duration", fmtMs(span.duration_ms)], ...optional.filter((fact): fact is Fact => fact !== null)]; @@ -42,6 +43,7 @@ export function SpanPane({ }: SpanTabProps & { trace: Trace; span: Span; accessToken: string; onClose: () => void }) { const { trace_id: traceId, trace_ref: traceRef, start_time: startTime } = trace.summary; const handoff = useTracesApi(accessToken).handoff(traceId, span.span_id, traceRef); + const traceStartMs = Date.parse(startTime); const detailQuery = useSpanDetail(accessToken, traceId, tab === "attributes" ? span.span_id : null, traceRef); return (