scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

fix(pass_through_endpoints): honor team/key logging callbacks on pass-through routes

Browse source 
Resolves LIT-5152

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
milan 2026-08-04 17:49:41 +00:00
parent cbeaf86c8d
commit ececda6574
4 changed files with 247 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
litellm/proxy/pass_through_endpoints/common_utils.py
View file

@ -1,5 +1,9 @@
from dataclasses import dataclass
from fastapi import Request
from litellm.proxy._types import UserAPIKeyAuth
def get_litellm_virtual_key(request: Request) -> str:
"""
@ -14,3 +18,42 @@ def get_litellm_virtual_key(request: Request) -> str:
if litellm_api_key:
return f"Bearer {litellm_api_key}"
return request.headers.get("Authorization", "")
@dataclass(frozen=True, slots=True)
class PassThroughDynamicLoggingParams:
"""Key/team-scoped logging settings, shaped for the ``Logging`` constructor"""
callback_vars: dict[str, str] | None
success_callbacks: list[str] | None
failure_callbacks: list[str] | None
NO_PASS_THROUGH_DYNAMIC_LOGGING = PassThroughDynamicLoggingParams(
callback_vars=None, success_callbacks=None, failure_callbacks=None
)
def get_pass_through_dynamic_logging_params(
user_api_key_dict: UserAPIKeyAuth,
) -> PassThroughDynamicLoggingParams:
"""
Resolve the key-level or team-level logging settings for a pass-through request.
``/chat/completions`` and friends get these through
``add_litellm_data_to_request``, which pass-through routes never call; without
this, a team's logging credentials are ignored and its traces land in whichever
project the global callback points at.
"""
from litellm.proxy.litellm_pre_call_utils import _get_dynamic_logging_metadata
from litellm.proxy.proxy_server import proxy_config
callback_settings = _get_dynamic_logging_metadata(user_api_key_dict=user_api_key_dict, proxy_config=proxy_config)
if callback_settings is None:
return NO_PASS_THROUGH_DYNAMIC_LOGGING
return PassThroughDynamicLoggingParams(
callback_vars=callback_settings.callback_vars,
success_callbacks=callback_settings.success_callback,
failure_callbacks=callback_settings.failure_callback,
)

11
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -67,6 +67,9 @@ from litellm.proxy.common_utils.http_parsing_utils import (
_safe_get_request_headers,
)
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.pass_through_endpoints.common_utils import (
get_pass_through_dynamic_logging_params,
)
from litellm.proxy.utils import normalize_route_for_root_path
from litellm.repositories.team_repository import TeamRepository
from litellm.secret_managers.main import get_secret_str
@ -897,6 +900,7 @@ async def pass_through_request(
# read e.g. ``chat gpt-4o`` instead of ``chat unknown``.
passthrough_model = (_parsed_body.get("model") if isinstance(_parsed_body, dict) else None) or "unknown"
start_time = datetime.now()
dynamic_logging = get_pass_through_dynamic_logging_params(user_api_key_dict=user_api_key_dict)
logging_obj = Logging(
model=passthrough_model,
messages=[{"role": "user", "content": safe_dumps(_parsed_body)}],
@ -905,6 +909,9 @@ async def pass_through_request(
start_time=start_time,
litellm_call_id=litellm_call_id,
function_id="1245",
kwargs=dynamic_logging.callback_vars,
dynamic_success_callbacks=dynamic_logging.success_callbacks,
dynamic_failure_callbacks=dynamic_logging.failure_callbacks,
)
# Store passthrough guardrails config on logging_obj for field targeting
@ -1910,6 +1917,7 @@ async def websocket_passthrough_request(
upstream_headers[header_name] = header_value
# Initialize logging object similar to HTTP passthrough
dynamic_logging = get_pass_through_dynamic_logging_params(user_api_key_dict=user_api_key_dict)
logging_obj = Logging(
model="unknown",
messages=[{"role": "user", "content": "WebSocket connection"}],
@ -1918,6 +1926,9 @@ async def websocket_passthrough_request(
start_time=start_time,
litellm_call_id=litellm_call_id,
function_id="websocket_passthrough",
kwargs=dynamic_logging.callback_vars,
dynamic_success_callbacks=dynamic_logging.success_callbacks,
dynamic_failure_callbacks=dynamic_logging.failure_callbacks,
)
# Create passthrough logging payload

145
tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -4877,3 +4877,148 @@ async def test_unusable_upstream_cost_records_zero_not_the_flat_estimate():
assert len(payloads) == 1
assert payloads[0]["response_cost"] == 0.0
assert payloads[0]["total_tokens"] == 1874
def _langfuse_otel_logging_metadata() -> dict:
return {
"logging": [
{
"callback_name": "langfuse_otel",
"callback_type": "success",
"callback_vars": {
"langfuse_public_key": "pk-team-project-2",
"langfuse_secret_key": "sk-team-project-2",
"langfuse_host": "https://team.langfuse.example",
},
}
]
}
async def _capture_pass_through_logging_obj(user_api_key_dict: UserAPIKeyAuth):
"""
Run a pass-through request and hand back the Logging object it built.
Streaming is used purely because chunk_processor is the cheapest place to
intercept that object.
"""
with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging:
with patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client"
) as mock_get_client:
with patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.PassThroughStreamingHandler.chunk_processor"
) as mock_chunk_processor:
mock_proxy_logging.pre_call_hook = AsyncMock(
return_value={"model": "gpt-4o-mini", "stream": True}
)
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mock_proxy_logging.post_call_response_headers_hook = AsyncMock(
return_value={}
)
upstream_response = MagicMock()
upstream_response.status_code = 200
upstream_response.headers = {}
upstream_response.raise_for_status = MagicMock()
async_client = MagicMock()
async_client.build_request = MagicMock(return_value=MagicMock())
async_client.send = AsyncMock(return_value=upstream_response)
mock_get_client.return_value = MagicMock(client=async_client)
async def _empty_chunks(*args, **kwargs):
return
yield # pragma: no cover
mock_chunk_processor.return_value = _empty_chunks()
mock_request = MagicMock(spec=Request)
mock_request.method = "POST"
mock_request.url = "http://test-proxy.com/openai/chat/completions"
mock_request.body = AsyncMock(
return_value=b'{"model": "gpt-4o-mini", "stream": true}'
)
mock_request.headers = Headers({})
mock_request.query_params = QueryParams({})
await pass_through_request(
request=mock_request,
target="https://api.openai.com/v1/chat/completions",
custom_headers={},
user_api_key_dict=user_api_key_dict,
stream=True,
)
return mock_chunk_processor.call_args.kwargs["litellm_logging_obj"]
@pytest.mark.asyncio
async def test_pass_through_request_logs_with_team_callback_credentials():
"""
Regression (LIT-5152): a pass-through request made with a team-scoped key must
log to the team's logging destination. Before this, pass-through routes never
resolved team callback settings, so their traces silently went to whichever
project the global callback pointed at while /chat/completions on the same key
went to the team's project.
"""
from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
logging_obj = await _capture_pass_through_logging_obj(
UserAPIKeyAuth(
api_key="sk-team-key",
team_id="team-project-2",
team_metadata=_langfuse_otel_logging_metadata(),
)
)
assert logging_obj.standard_callback_dynamic_params == {
"langfuse_public_key": "pk-team-project-2",
"langfuse_secret_key": "sk-team-project-2",
"langfuse_host": "https://team.langfuse.example",
}
assert any(
isinstance(callback, LangfuseOtelLogger)
for callback in logging_obj.dynamic_async_success_callbacks or []
)
@pytest.mark.asyncio
async def test_pass_through_request_logs_with_key_callback_credentials():
"""Key-level logging settings win over the team's, same as on /chat/completions"""
logging_obj = await _capture_pass_through_logging_obj(
UserAPIKeyAuth(
api_key="sk-key-with-own-logging",
team_id="team-project-2",
team_metadata=_langfuse_otel_logging_metadata(),
metadata={
"logging": [
{
"callback_name": "langfuse_otel",
"callback_type": "success",
"callback_vars": {
"langfuse_public_key": "pk-key-project-3",
"langfuse_secret_key": "sk-key-project-3",
},
}
]
},
)
)
assert logging_obj.standard_callback_dynamic_params == {
"langfuse_public_key": "pk-key-project-3",
"langfuse_secret_key": "sk-key-project-3",
}
@pytest.mark.asyncio
async def test_pass_through_request_without_scoped_logging_settings_uses_global_callbacks():
"""A key with no logging settings of its own must not pick up per-request credentials"""
logging_obj = await _capture_pass_through_logging_obj(
UserAPIKeyAuth(api_key="sk-plain-key")
)
assert logging_obj.standard_callback_dynamic_params == {}
assert logging_obj.dynamic_success_callbacks is None
assert logging_obj.dynamic_async_success_callbacks is None

48
tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoints_common_utils.py
View file

@ -97,3 +97,51 @@ def test_encode_bedrock_runtime_modelid_arn_edge_cases():
expected = "model/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile%2Ftest-profile.v1/invoke"
result = CommonUtils.encode_bedrock_runtime_modelid_arn(endpoint)
assert result == expected
def test_get_pass_through_dynamic_logging_params_from_default_team_settings():
"""
Regression (LIT-5152): pass-through routes must honor `default_team_settings`
from config.yaml, the shape the original report (BerriAI/litellm#9967) used
"""
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.pass_through_endpoints.common_utils import (
get_pass_through_dynamic_logging_params,
)
proxy_config = Mock()
proxy_config.load_team_config.return_value = {
"success_callback": ["langfuse"],
"langfuse_public_key": "pk-team-project-2",
"langfuse_secret": "sk-team-project-2",
}
with patch("litellm.proxy.proxy_server.proxy_config", proxy_config):
params = get_pass_through_dynamic_logging_params(
user_api_key_dict=UserAPIKeyAuth(api_key="sk-team-key", team_id="team-project-2")
)
assert params.success_callbacks == ["langfuse"]
assert params.callback_vars == {
"langfuse_public_key": "pk-team-project-2",
"langfuse_secret": "sk-team-project-2",
}
def test_get_pass_through_dynamic_logging_params_without_settings():
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.pass_through_endpoints.common_utils import (
get_pass_through_dynamic_logging_params,
)
proxy_config = Mock()
proxy_config.load_team_config.return_value = {}
with patch("litellm.proxy.proxy_server.proxy_config", proxy_config):
params = get_pass_through_dynamic_logging_params(
user_api_key_dict=UserAPIKeyAuth(api_key="sk-plain-key", team_id="team-no-logging")
)
assert params.callback_vars is None
assert params.success_callbacks is None
assert params.failure_callbacks is None