diff --git a/litellm/proxy/pass_through_endpoints/common_utils.py b/litellm/proxy/pass_through_endpoints/common_utils.py index 3a3783dd57c..be4f791aca3 100644 --- a/litellm/proxy/pass_through_endpoints/common_utils.py +++ b/litellm/proxy/pass_through_endpoints/common_utils.py @@ -1,5 +1,9 @@ +from dataclasses import dataclass + from fastapi import Request +from litellm.proxy._types import UserAPIKeyAuth + def get_litellm_virtual_key(request: Request) -> str: """ @@ -14,3 +18,42 @@ def get_litellm_virtual_key(request: Request) -> str: if litellm_api_key: return f"Bearer {litellm_api_key}" return request.headers.get("Authorization", "") + + +@dataclass(frozen=True, slots=True) +class PassThroughDynamicLoggingParams: + """Key/team-scoped logging settings, shaped for the ``Logging`` constructor""" + + callback_vars: dict[str, str] | None + success_callbacks: list[str] | None + failure_callbacks: list[str] | None + + +NO_PASS_THROUGH_DYNAMIC_LOGGING = PassThroughDynamicLoggingParams( + callback_vars=None, success_callbacks=None, failure_callbacks=None +) + + +def get_pass_through_dynamic_logging_params( + user_api_key_dict: UserAPIKeyAuth, +) -> PassThroughDynamicLoggingParams: + """ + Resolve the key-level or team-level logging settings for a pass-through request. + + ``/chat/completions`` and friends get these through + ``add_litellm_data_to_request``, which pass-through routes never call; without + this, a team's logging credentials are ignored and its traces land in whichever + project the global callback points at. + """ + from litellm.proxy.litellm_pre_call_utils import _get_dynamic_logging_metadata + from litellm.proxy.proxy_server import proxy_config + + callback_settings = _get_dynamic_logging_metadata(user_api_key_dict=user_api_key_dict, proxy_config=proxy_config) + if callback_settings is None: + return NO_PASS_THROUGH_DYNAMIC_LOGGING + + return PassThroughDynamicLoggingParams( + callback_vars=callback_settings.callback_vars, + success_callbacks=callback_settings.success_callback, + failure_callbacks=callback_settings.failure_callback, + ) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 171df5b4d2c..8a002259656 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -67,6 +67,9 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _safe_get_request_headers, ) from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.pass_through_endpoints.common_utils import ( + get_pass_through_dynamic_logging_params, +) from litellm.proxy.utils import normalize_route_for_root_path from litellm.repositories.team_repository import TeamRepository from litellm.secret_managers.main import get_secret_str @@ -897,6 +900,7 @@ async def pass_through_request( # read e.g. ``chat gpt-4o`` instead of ``chat unknown``. passthrough_model = (_parsed_body.get("model") if isinstance(_parsed_body, dict) else None) or "unknown" start_time = datetime.now() + dynamic_logging = get_pass_through_dynamic_logging_params(user_api_key_dict=user_api_key_dict) logging_obj = Logging( model=passthrough_model, messages=[{"role": "user", "content": safe_dumps(_parsed_body)}], @@ -905,6 +909,9 @@ async def pass_through_request( start_time=start_time, litellm_call_id=litellm_call_id, function_id="1245", + kwargs=dynamic_logging.callback_vars, + dynamic_success_callbacks=dynamic_logging.success_callbacks, + dynamic_failure_callbacks=dynamic_logging.failure_callbacks, ) # Store passthrough guardrails config on logging_obj for field targeting @@ -1910,6 +1917,7 @@ async def websocket_passthrough_request( upstream_headers[header_name] = header_value # Initialize logging object similar to HTTP passthrough + dynamic_logging = get_pass_through_dynamic_logging_params(user_api_key_dict=user_api_key_dict) logging_obj = Logging( model="unknown", messages=[{"role": "user", "content": "WebSocket connection"}], @@ -1918,6 +1926,9 @@ async def websocket_passthrough_request( start_time=start_time, litellm_call_id=litellm_call_id, function_id="websocket_passthrough", + kwargs=dynamic_logging.callback_vars, + dynamic_success_callbacks=dynamic_logging.success_callbacks, + dynamic_failure_callbacks=dynamic_logging.failure_callbacks, ) # Create passthrough logging payload diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 9bddeda0723..c19d0d0ffec 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -4877,3 +4877,148 @@ async def test_unusable_upstream_cost_records_zero_not_the_flat_estimate(): assert len(payloads) == 1 assert payloads[0]["response_cost"] == 0.0 assert payloads[0]["total_tokens"] == 1874 + + +def _langfuse_otel_logging_metadata() -> dict: + return { + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-team-project-2", + "langfuse_secret_key": "sk-team-project-2", + "langfuse_host": "https://team.langfuse.example", + }, + } + ] + } + + +async def _capture_pass_through_logging_obj(user_api_key_dict: UserAPIKeyAuth): + """ + Run a pass-through request and hand back the Logging object it built. + + Streaming is used purely because chunk_processor is the cheapest place to + intercept that object. + """ + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client" + ) as mock_get_client: + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.PassThroughStreamingHandler.chunk_processor" + ) as mock_chunk_processor: + mock_proxy_logging.pre_call_hook = AsyncMock( + return_value={"model": "gpt-4o-mini", "stream": True} + ) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock( + return_value={} + ) + + upstream_response = MagicMock() + upstream_response.status_code = 200 + upstream_response.headers = {} + upstream_response.raise_for_status = MagicMock() + + async_client = MagicMock() + async_client.build_request = MagicMock(return_value=MagicMock()) + async_client.send = AsyncMock(return_value=upstream_response) + mock_get_client.return_value = MagicMock(client=async_client) + + async def _empty_chunks(*args, **kwargs): + return + yield # pragma: no cover + + mock_chunk_processor.return_value = _empty_chunks() + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = "http://test-proxy.com/openai/chat/completions" + mock_request.body = AsyncMock( + return_value=b'{"model": "gpt-4o-mini", "stream": true}' + ) + mock_request.headers = Headers({}) + mock_request.query_params = QueryParams({}) + + await pass_through_request( + request=mock_request, + target="https://api.openai.com/v1/chat/completions", + custom_headers={}, + user_api_key_dict=user_api_key_dict, + stream=True, + ) + + return mock_chunk_processor.call_args.kwargs["litellm_logging_obj"] + + +@pytest.mark.asyncio +async def test_pass_through_request_logs_with_team_callback_credentials(): + """ + Regression (LIT-5152): a pass-through request made with a team-scoped key must + log to the team's logging destination. Before this, pass-through routes never + resolved team callback settings, so their traces silently went to whichever + project the global callback pointed at while /chat/completions on the same key + went to the team's project. + """ + from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger + + logging_obj = await _capture_pass_through_logging_obj( + UserAPIKeyAuth( + api_key="sk-team-key", + team_id="team-project-2", + team_metadata=_langfuse_otel_logging_metadata(), + ) + ) + + assert logging_obj.standard_callback_dynamic_params == { + "langfuse_public_key": "pk-team-project-2", + "langfuse_secret_key": "sk-team-project-2", + "langfuse_host": "https://team.langfuse.example", + } + assert any( + isinstance(callback, LangfuseOtelLogger) + for callback in logging_obj.dynamic_async_success_callbacks or [] + ) + + +@pytest.mark.asyncio +async def test_pass_through_request_logs_with_key_callback_credentials(): + """Key-level logging settings win over the team's, same as on /chat/completions""" + logging_obj = await _capture_pass_through_logging_obj( + UserAPIKeyAuth( + api_key="sk-key-with-own-logging", + team_id="team-project-2", + team_metadata=_langfuse_otel_logging_metadata(), + metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-key-project-3", + "langfuse_secret_key": "sk-key-project-3", + }, + } + ] + }, + ) + ) + + assert logging_obj.standard_callback_dynamic_params == { + "langfuse_public_key": "pk-key-project-3", + "langfuse_secret_key": "sk-key-project-3", + } + + +@pytest.mark.asyncio +async def test_pass_through_request_without_scoped_logging_settings_uses_global_callbacks(): + """A key with no logging settings of its own must not pick up per-request credentials""" + logging_obj = await _capture_pass_through_logging_obj( + UserAPIKeyAuth(api_key="sk-plain-key") + ) + + assert logging_obj.standard_callback_dynamic_params == {} + assert logging_obj.dynamic_success_callbacks is None + assert logging_obj.dynamic_async_success_callbacks is None diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py index 797b22784ae..7019f62c682 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py @@ -97,3 +97,51 @@ def test_encode_bedrock_runtime_modelid_arn_edge_cases(): expected = "model/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile%2Ftest-profile.v1/invoke" result = CommonUtils.encode_bedrock_runtime_modelid_arn(endpoint) assert result == expected + + +def test_get_pass_through_dynamic_logging_params_from_default_team_settings(): + """ + Regression (LIT-5152): pass-through routes must honor `default_team_settings` + from config.yaml, the shape the original report (BerriAI/litellm#9967) used + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.pass_through_endpoints.common_utils import ( + get_pass_through_dynamic_logging_params, + ) + + proxy_config = Mock() + proxy_config.load_team_config.return_value = { + "success_callback": ["langfuse"], + "langfuse_public_key": "pk-team-project-2", + "langfuse_secret": "sk-team-project-2", + } + + with patch("litellm.proxy.proxy_server.proxy_config", proxy_config): + params = get_pass_through_dynamic_logging_params( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-team-key", team_id="team-project-2") + ) + + assert params.success_callbacks == ["langfuse"] + assert params.callback_vars == { + "langfuse_public_key": "pk-team-project-2", + "langfuse_secret": "sk-team-project-2", + } + + +def test_get_pass_through_dynamic_logging_params_without_settings(): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.pass_through_endpoints.common_utils import ( + get_pass_through_dynamic_logging_params, + ) + + proxy_config = Mock() + proxy_config.load_team_config.return_value = {} + + with patch("litellm.proxy.proxy_server.proxy_config", proxy_config): + params = get_pass_through_dynamic_logging_params( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-plain-key", team_id="team-no-logging") + ) + + assert params.callback_vars is None + assert params.success_callbacks is None + assert params.failure_callbacks is None