From b4f9e14a44789b408c3ceab56d01671f33a7b3d0 Mon Sep 17 00:00:00 2001 From: milan Date: Tue, 28 Jul 2026 14:14:53 +0000 Subject: [PATCH 1/8] feat(dd_span_tagger): emit litellm_user_email span tag for JWT-authenticated requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/dd_span_tagger.py | 8 +++++- .../proxy/test_common_request_processing.py | 27 ++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/dd_span_tagger.py b/litellm/proxy/dd_span_tagger.py index 08b7d928d0e..1fda0c638d4 100644 --- a/litellm/proxy/dd_span_tagger.py +++ b/litellm/proxy/dd_span_tagger.py @@ -38,11 +38,15 @@ class DDSpanTagger: - ``litellm.key_alias`` — human-readable alias for the API key - ``litellm.key_hash`` — hashed API key (safe to log; never the raw secret) - ``litellm.requested_model``— model name as sent by the client + - ``litellm_user_email`` — email of the authenticated user Use cases: - Trace all requests from a specific user/key: filter by ``litellm.key_alias`` or ``litellm.key_hash``. - Trace all requests for a specific model: filter by ``litellm.requested_model``. + - Trace all requests from a specific person under JWT auth, where there is no key_alias: + filter by ``@litellm_user_email:"user@example.com"``. The value comes from the JWT claim + mapped by ``litellm_jwtauth.user_email_jwt_field``, or from the user row for virtual keys. Note: key_alias / key_hash are not available for unauthenticated (e.g. 401) requests. """ @@ -53,8 +57,10 @@ class DDSpanTagger: set_active_span_tag("litellm.key_hash", str(user_api_key_dict.token)) if requested_model: set_active_span_tag("litellm.requested_model", str(requested_model)) + if user_api_key_dict.user_email: + set_active_span_tag("litellm_user_email", str(user_api_key_dict.user_email)) except Exception: verbose_proxy_logger.debug( - "Failed to tag active ddtrace span with key/model tags", + "Failed to tag active ddtrace span with key/model/user tags", exc_info=True, ) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 58f81cdad35..287e6f2f80c 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -2318,12 +2318,13 @@ class TestStreamingOverheadHeader: class TestDDSpanTaggerTagRequest: """Tests for DDSpanTagger.tag_request - key/model DD span tagging.""" - def _make_user_api_key_dict(self, key_alias=None, token=None): + def _make_user_api_key_dict(self, key_alias=None, token=None, user_email=None): from litellm.proxy._types import UserAPIKeyAuth d = UserAPIKeyAuth() d.key_alias = key_alias d.token = token + d.user_email = user_email return d def test_tags_key_alias_and_model(self): @@ -2364,6 +2365,30 @@ class TestDDSpanTaggerTagRequest: mock_set_tag.assert_called_once_with("litellm.requested_model", "claude-3-5-sonnet") + def test_tags_user_email(self): + """user_email is tagged so JWT-authenticated requests are traceable per person.""" + user_key = self._make_user_api_key_dict(user_email="user@example.com") + + with patch("litellm.proxy.dd_span_tagger.set_active_span_tag") as mock_set_tag: + DDSpanTagger.tag_request( + user_api_key_dict=user_key, + requested_model=None, + ) + + mock_set_tag.assert_called_once_with("litellm_user_email", "user@example.com") + + def test_no_user_email_tag_when_absent(self): + """No user email tag when the authenticated identity has no email.""" + user_key = self._make_user_api_key_dict(key_alias="my-prod-key", user_email=None) + + with patch("litellm.proxy.dd_span_tagger.set_active_span_tag") as mock_set_tag: + DDSpanTagger.tag_request( + user_api_key_dict=user_key, + requested_model="gpt-4o", + ) + + assert all(call.args[0] != "litellm_user_email" for call in mock_set_tag.call_args_list) + class TestHasAttributeErrorInChain: """Tests for _has_attribute_error_in_chain helper.""" From 819faa01c336d825619b3592ed1c1d763a4156d4 Mon Sep 17 00:00:00 2001 From: milan Date: Tue, 28 Jul 2026 14:40:04 +0000 Subject: [PATCH 2/8] refactor(dd_span_tagger): use dotted litellm.user_email tag for consistency Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/dd_span_tagger.py | 6 +++--- tests/test_litellm/proxy/test_common_request_processing.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/dd_span_tagger.py b/litellm/proxy/dd_span_tagger.py index 1fda0c638d4..cafae9bb2a3 100644 --- a/litellm/proxy/dd_span_tagger.py +++ b/litellm/proxy/dd_span_tagger.py @@ -38,14 +38,14 @@ class DDSpanTagger: - ``litellm.key_alias`` — human-readable alias for the API key - ``litellm.key_hash`` — hashed API key (safe to log; never the raw secret) - ``litellm.requested_model``— model name as sent by the client - - ``litellm_user_email`` — email of the authenticated user + - ``litellm.user_email`` — email of the authenticated user Use cases: - Trace all requests from a specific user/key: filter by ``litellm.key_alias`` or ``litellm.key_hash``. - Trace all requests for a specific model: filter by ``litellm.requested_model``. - Trace all requests from a specific person under JWT auth, where there is no key_alias: - filter by ``@litellm_user_email:"user@example.com"``. The value comes from the JWT claim + filter by ``@litellm.user_email:"user@example.com"``. The value comes from the JWT claim mapped by ``litellm_jwtauth.user_email_jwt_field``, or from the user row for virtual keys. Note: key_alias / key_hash are not available for unauthenticated (e.g. 401) requests. @@ -58,7 +58,7 @@ class DDSpanTagger: if requested_model: set_active_span_tag("litellm.requested_model", str(requested_model)) if user_api_key_dict.user_email: - set_active_span_tag("litellm_user_email", str(user_api_key_dict.user_email)) + set_active_span_tag("litellm.user_email", str(user_api_key_dict.user_email)) except Exception: verbose_proxy_logger.debug( "Failed to tag active ddtrace span with key/model/user tags", diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 287e6f2f80c..312c3d2a884 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -2375,7 +2375,7 @@ class TestDDSpanTaggerTagRequest: requested_model=None, ) - mock_set_tag.assert_called_once_with("litellm_user_email", "user@example.com") + mock_set_tag.assert_called_once_with("litellm.user_email", "user@example.com") def test_no_user_email_tag_when_absent(self): """No user email tag when the authenticated identity has no email.""" @@ -2387,7 +2387,7 @@ class TestDDSpanTaggerTagRequest: requested_model="gpt-4o", ) - assert all(call.args[0] != "litellm_user_email" for call in mock_set_tag.call_args_list) + assert all(call.args[0] != "litellm.user_email" for call in mock_set_tag.call_args_list) class TestHasAttributeErrorInChain: From b224b15b9d24c15590a81c3c4bb5399d40b7d9e0 Mon Sep 17 00:00:00 2001 From: milan Date: Fri, 21 Aug 2026 02:41:55 +0000 Subject: [PATCH 3/8] feat(model_hub): surface model_info.description in model group info and Model Hub UI Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 6 +++++ litellm/types/router.py | 1 + tests/test_litellm/test_router.py | 25 +++++++++++++++++++ .../src/components/AIHub/ModelHubTable.tsx | 6 +++++ .../components/AIHub/ModelHubTableColumns.tsx | 1 + .../components/PublicModelHubTableColumns.tsx | 1 + .../src/components/public_model_hub.tsx | 6 +++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 ++ 8 files changed, 48 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index e9eeab53934..90968800460 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9334,6 +9334,9 @@ class Router: model_litellm_params = model.get("litellm_params", {}) model_info_dict = model.get("model_info", {}) + _raw_description = model_info_dict.get("description") + _deployment_description: str | None = _raw_description if isinstance(_raw_description, str) else None + # get model tpm _deployment_tpm: int | None = None if _deployment_tpm is None: @@ -9415,6 +9418,7 @@ class Router: **{ "model_group": user_facing_model_group_name, "providers": [llm_provider], + "description": _deployment_description, **model_info, } ) @@ -9428,6 +9432,8 @@ class Router: # supports_function_calling == True if llm_provider not in model_group_info.providers: model_group_info.providers.append(llm_provider) + if model_group_info.description is None and _deployment_description is not None: + model_group_info.description = _deployment_description if ( model_info.get("max_input_tokens", None) is not None and model_info["max_input_tokens"] is not None diff --git a/litellm/types/router.py b/litellm/types/router.py index 99a4603ae49..b33e387c508 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -637,6 +637,7 @@ class ModelGroupInfo(BaseModel): supports_function_calling: bool = Field(default=False) supported_openai_params: list[str] | None = Field(default=[]) configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None + description: str | None = None def __init__(self, **data) -> None: for field_name, field_type in get_type_hints(self.__class__).items(): diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 9894fcef163..765e479e378 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1626,6 +1626,31 @@ def test_model_group_info_cost_none_when_db_model_info_has_no_cost(): assert result.output_cost_per_token is None +def test_model_group_info_description_from_model_info(): + """ + model_info.description set in the config should surface on ModelGroupInfo, + including when only a later deployment in the group carries it. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4", "api_key": "fake"}, + "model_info": {}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4", "api_key": "fake2"}, + "model_info": {"description": "State-of-the-art language model."}, + }, + ] + ) + + result = router.get_model_group_info("gpt-4") + assert result is not None + assert result.description == "State-of-the-art language model." + + @pytest.mark.parametrize( "value,expected", [ diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index 299104b271b..5fadbe3300d 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -647,6 +647,12 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, ))} + {selectedModel.description && ( +
+

Description:

+

{selectedModel.description}

+
+ )} diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx index 9f74771f3b1..b025bcf62aa 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx @@ -31,6 +31,7 @@ export interface ModelHubData { supports_function_calling: boolean; supported_openai_params?: string[]; is_public_model_group: boolean; + description?: string; [key: string]: any; } diff --git a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx index ab0ed976149..f3673df8025 100644 --- a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx @@ -24,6 +24,7 @@ export interface ModelGroupInfo { health_status?: string; health_response_time?: number; health_checked_at?: string; + description?: string; [key: string]: any; } diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index 171b7992325..cd10068b9e6 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -918,6 +918,12 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded })} + {selectedModel.description && ( +
+

Description:

+

{selectedModel.description}

+
+ )} {/* Wildcard Routing Note */} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index fff48e14ecf..39b3255efb3 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29481,6 +29481,8 @@ export interface components { ModelGroupInfoProxy: { /** Configurable Clientside Auth Params */ configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Output"])[] | null; + /** Description */ + description?: string | null; /** Health Checked At */ health_checked_at?: string | null; /** Health Response Time */ From 342b93fe0a92f32b8c2fb4ccf61441cb0a2cf3b1 Mon Sep 17 00:00:00 2001 From: milan Date: Fri, 21 Aug 2026 03:27:35 +0000 Subject: [PATCH 4/8] refactor(router): aggregate model group description without in-place mutation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 90968800460..d65c13d57b2 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9313,6 +9313,10 @@ class Router: model_list: Final = self.get_model_list(model_name=model_group) if model_list is None: return None + model_group_description: Final[str | None] = next( + (d for d in (m.get("model_info", {}).get("description") for m in model_list) if isinstance(d, str)), + None, + ) for model in model_list: is_match = False if ( @@ -9334,9 +9338,6 @@ class Router: model_litellm_params = model.get("litellm_params", {}) model_info_dict = model.get("model_info", {}) - _raw_description = model_info_dict.get("description") - _deployment_description: str | None = _raw_description if isinstance(_raw_description, str) else None - # get model tpm _deployment_tpm: int | None = None if _deployment_tpm is None: @@ -9418,7 +9419,7 @@ class Router: **{ "model_group": user_facing_model_group_name, "providers": [llm_provider], - "description": _deployment_description, + "description": model_group_description, **model_info, } ) @@ -9432,8 +9433,6 @@ class Router: # supports_function_calling == True if llm_provider not in model_group_info.providers: model_group_info.providers.append(llm_provider) - if model_group_info.description is None and _deployment_description is not None: - model_group_info.description = _deployment_description if ( model_info.get("max_input_tokens", None) is not None and model_info["max_input_tokens"] is not None From 6d2c4899b0f9fa36c59c793657234f75b6a83704 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 13 Sep 2026 09:44:22 +0000 Subject: [PATCH 5/8] fix(health): skip background health check DB writes when the latest-row read fails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/db/health_check_latest.py | 8 ++++-- .../health_endpoints/_health_endpoints.py | 7 +++-- .../proxy/db/test_health_check_latest.py | 10 +++++++ .../proxy/test_health_check_functions.py | 27 +++++++++++++++---- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/db/health_check_latest.py b/litellm/proxy/db/health_check_latest.py index 35bc838379c..21438f095bb 100644 --- a/litellm/proxy/db/health_check_latest.py +++ b/litellm/proxy/db/health_check_latest.py @@ -74,10 +74,14 @@ class LatestHealthCheckRow(BaseModel): _ROWS_ADAPTER: Final = TypeAdapter(tuple[LatestHealthCheckRow, ...]) +async def query_latest_health_checks(prisma_client: PrismaClient) -> tuple[LatestHealthCheckRow, ...]: + rows: Final = await prisma_client.db.query_raw(LATEST_HEALTH_CHECKS_SQL) + return _ROWS_ADAPTER.validate_python(rows) + + async def fetch_latest_health_checks(prisma_client: PrismaClient) -> tuple[LatestHealthCheckRow, ...]: try: - rows: Final = await prisma_client.db.query_raw(LATEST_HEALTH_CHECKS_SQL) - return _ROWS_ADAPTER.validate_python(rows) + return await query_latest_health_checks(prisma_client) except Exception as query_err: # noqa: BLE001 # health decorates other reads; a driver error must not fail them verbose_proxy_logger.error("Error getting all latest health checks: %s", query_err) return () diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index db6ec754c6e..64fd59bbe44 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -45,7 +45,10 @@ from litellm.proxy.auth.auth_utils import ( from litellm.proxy.auth.model_checks import get_key_models from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler -from litellm.proxy.db.health_check_latest import LatestHealthCheckRow +from litellm.proxy.db.health_check_latest import ( + LatestHealthCheckRow, + query_latest_health_checks, +) from litellm.proxy.db.proxy_worker_heartbeat import count_live_proxy_workers from litellm.proxy.health_check import ( ADMIN_ONLY_HEALTH_DISPLAY_PARAMS, @@ -876,7 +879,7 @@ async def _save_background_health_checks_to_db( ) # Step 3: Get latest health checks for all models in one query to compare status - latest_checks: Final = await prisma_client.get_all_latest_health_checks() + latest_checks: Final = await query_latest_health_checks(prisma_client) latest_checks_map: Final = {} for check in latest_checks: # Use model_id as primary key, fallback to model_name diff --git a/tests/test_litellm/proxy/db/test_health_check_latest.py b/tests/test_litellm/proxy/db/test_health_check_latest.py index 529e4d8f2e9..6322891ae9e 100644 --- a/tests/test_litellm/proxy/db/test_health_check_latest.py +++ b/tests/test_litellm/proxy/db/test_health_check_latest.py @@ -8,6 +8,7 @@ from litellm.proxy.db.health_check_latest import ( LATEST_HEALTH_CHECKS_SQL, fetch_latest_health_checks, fetch_latest_health_checks_for_models, + query_latest_health_checks, ) @@ -83,6 +84,15 @@ async def test_fetch_all_degrades_to_no_rows_when_the_query_fails(): assert await fetch_latest_health_checks(prisma) == () +@pytest.mark.asyncio +async def test_query_all_raises_when_the_query_fails_instead_of_reading_as_an_empty_table(): + """The background save decides what to write from this read; a failure has to be told apart from no rows.""" + prisma = _prisma([]) + prisma.db.query_raw.side_effect = RuntimeError("db down") + with pytest.raises(RuntimeError, match="db down"): + await query_latest_health_checks(prisma) + + @pytest.mark.asyncio async def test_fetch_all_degrades_to_no_rows_for_a_malformed_row(): assert await fetch_latest_health_checks(_prisma([{"unexpected": "shape"}])) == () diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index c0c853ae2c5..7d6c5d3cebe 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -374,7 +374,7 @@ async def test_save_background_health_checks_to_db(): """Test the main background health check save function""" mock_prisma = MagicMock() mock_prisma.save_health_check_result = AsyncMock() - mock_prisma.get_all_latest_health_checks = AsyncMock(return_value=[]) + mock_prisma.db.query_raw = AsyncMock(return_value=[]) model_list = [ { @@ -398,9 +398,9 @@ async def test_save_background_health_checks_to_db(): "background_health_check", ) - # Should call get_all_latest_health_checks and save_health_check_result, and report completion + # Should read the latest rows and save_health_check_result, and report completion assert persisted is True - mock_prisma.get_all_latest_health_checks.assert_called_once() + mock_prisma.db.query_raw.assert_awaited_once() mock_prisma.save_health_check_result.assert_called_once() call_kwargs = mock_prisma.save_health_check_result.call_args[1] @@ -493,7 +493,7 @@ def _one_model_setup(): @pytest.mark.asyncio async def test_save_background_health_checks_to_db_returns_false_when_a_write_fails(): mock_prisma = MagicMock() - mock_prisma.get_all_latest_health_checks = AsyncMock(return_value=[]) + mock_prisma.db.query_raw = AsyncMock(return_value=[]) mock_prisma.save_health_check_result = AsyncMock(return_value=None) model_list, healthy_endpoints, unhealthy_endpoints = _one_model_setup() @@ -504,6 +504,23 @@ async def test_save_background_health_checks_to_db_returns_false_when_a_write_fa assert (persisted, mock_prisma.save_health_check_result.await_count) == (False, 1) +@pytest.mark.asyncio +async def test_save_background_health_checks_to_db_writes_nothing_when_the_latest_row_read_fails(mock_prisma): + """ + A failed dedup read must not read as an empty table. Treated that way, every model was written on every + cycle by every pod while the read kept failing, which is what filled the table in production. + """ + mock_prisma.db.query_raw = AsyncMock(side_effect=RuntimeError("db down")) + mock_prisma.save_health_check_result = AsyncMock(return_value={"id": "row"}) + model_list, healthy_endpoints, unhealthy_endpoints = _one_model_setup() + + persisted = await _save_background_health_checks_to_db( + mock_prisma, model_list, healthy_endpoints, unhealthy_endpoints, 1234567890.0, "background_health_check" + ) + + assert (persisted, mock_prisma.save_health_check_result.await_count) == (False, 0) + + @pytest.mark.asyncio async def test_save_background_health_checks_to_db_no_prisma(): """Test graceful handling when no prisma client""" @@ -515,7 +532,7 @@ async def test_save_background_health_checks_to_db_no_prisma(): async def test_save_background_health_checks_to_db_exception_handling(): """Test exception handling in background health check save""" mock_prisma = MagicMock() - mock_prisma.get_all_latest_health_checks = AsyncMock(side_effect=Exception("DB Error")) + mock_prisma.db.query_raw = AsyncMock(side_effect=Exception("DB Error")) model_list = [ { From 11a3499d4d4cbddd79b0bef19761aaa611997dd3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 11 Sep 2026 02:04:01 +0000 Subject: [PATCH 6/8] fix(openai): keep extra_headers off the image generation JSON body Proxy credentials put extra_headers on optional_params and extra_body. Pop them out of the Images request body and pass them only as the SDK header kwarg so OpenAI no longer returns Unknown parameter extra_headers. Co-authored-by: Zsanz3 --- litellm/llms/openai/openai.py | 54 ++++++-- ...t_openai_image_generation_extra_headers.py | 123 ++++++++++++++---- 2 files changed, 144 insertions(+), 33 deletions(-) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 5f04ebe0c01..0c6cff5bca5 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -340,6 +340,12 @@ _NO_EXTRA_HEADERS: Final[Mapping[str, str]] = types.MappingProxyType({}) _SDK_OPTION_KEYS: Final = frozenset(("extra_headers", "extra_query", "extra_body")) +def _as_mapping(value: object) -> Mapping[str, object]: + if not isinstance(value, dict): + return types.MappingProxyType({}) + return types.MappingProxyType({key: item for key, item in value.items()}) + + def _embedding_request_without_sdk_defaults( data: Mapping[str, object], timeout: float | httpx.Timeout ) -> tuple[Mapping[str, object], RequestOptions]: @@ -356,6 +362,37 @@ def _embedding_request_without_sdk_defaults( return body, options +def _image_generation_request_data( + data: Mapping[str, object], headers: Mapping[str, object] | None +) -> tuple[dict[str, object], Mapping[str, object]]: + extra_body_source: Final = _as_mapping(data.get("extra_body")) + extra_body: Final = types.MappingProxyType( + {key: item for key, item in extra_body_source.items() if key != "extra_headers"} + ) + extra_headers: Final = types.MappingProxyType( + { + **_as_mapping(extra_body_source.get("extra_headers")), + **_as_mapping(data.get("extra_headers")), + **_as_mapping(headers), + } + ) + extra_query: Final = _as_mapping(data.get("extra_query")) + logged_body: Final = { # mutable-ok: loggers isinstance-check this payload as a dict + key: item for key, item in data.items() if key not in _SDK_OPTION_KEYS + } + request_data: Final = types.MappingProxyType( + dict( + ( + *logged_body.items(), + *((("extra_headers", extra_headers),) if extra_headers else ()), + *((("extra_query", extra_query),) if extra_query else ()), + *((("extra_body", extra_body),) if extra_body else ()), + ) + ) + ) + return logged_body, request_data + + class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): def __init__(self) -> None: super().__init__() @@ -1415,6 +1452,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): client=client, ) + logged_body, request_data = _image_generation_request_data(data, headers) logging_obj.pre_call( input=prompt, api_key=openai_aclient.api_key, @@ -1422,20 +1460,17 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): "headers": {"Authorization": f"Bearer {openai_aclient.api_key}"}, # mutable-ok: logged header map "api_base": str(openai_aclient.base_url), "acompletion": True, - "complete_input_dict": data, + "complete_input_dict": logged_body, }, ) - request_data: Final = ( # mutable-ok: the OpenAI SDK takes the request body as a dict - {**data, "extra_headers": headers} if headers else data - ) response = await openai_aclient.images.generate(**request_data, timeout=timeout) stringified_response: Final = response.model_dump() ## LOGGING logging_obj.post_call( input=prompt, api_key=api_key, - additional_args={"complete_input_dict": data}, + additional_args={"complete_input_dict": logged_body}, original_response=stringified_response, ) return convert_to_model_response_object( @@ -1499,6 +1534,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): client=client, ) + logged_body, request_data = _image_generation_request_data(data, headers) + ## LOGGING logging_obj.pre_call( input=prompt, @@ -1507,14 +1544,11 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): "headers": {"Authorization": f"Bearer {openai_client.api_key}"}, "api_base": openai_client._base_url._uri_reference, "acompletion": True, - "complete_input_dict": data, + "complete_input_dict": logged_body, }, ) ## COMPLETION CALL - request_data: Final = ( # mutable-ok: the OpenAI SDK takes the request body as a dict - {**data, "extra_headers": headers} if headers else data - ) _response: Final = openai_client.images.generate(**request_data, timeout=timeout) response: Final = _response.model_dump() @@ -1522,7 +1556,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): logging_obj.post_call( input=prompt, api_key=api_key, - additional_args={"complete_input_dict": data}, + additional_args={"complete_input_dict": logged_body}, original_response=response, ) return convert_to_model_response_object( diff --git a/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py b/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py index 55ef74abd7b..b422e62b9c2 100644 --- a/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py +++ b/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py @@ -6,11 +6,10 @@ litellm.aimage_generation() are forwarded to the OpenAI API client as extra_headers in the images.generate() call. """ -from unittest.mock import MagicMock, AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest - from litellm.llms.openai.openai import OpenAIChatCompletion @@ -30,9 +29,7 @@ def mock_logging_obj(): class TestImageGenerationExtraHeaders: """Test that extra_headers are properly injected into OpenAI image generation calls.""" - def test_sync_image_generation_with_headers( - self, openai_chat_completions, mock_logging_obj - ): + def test_sync_image_generation_with_headers(self, openai_chat_completions, mock_logging_obj): """Sync image_generation should pass headers as extra_headers to images.generate().""" mock_image_data = MagicMock() mock_image_data.model_dump.return_value = { @@ -61,9 +58,7 @@ class TestImageGenerationExtraHeaders: _, kwargs = mock_openai_client.images.generate.call_args assert kwargs.get("extra_headers") == test_headers - def test_sync_image_generation_without_headers( - self, openai_chat_completions, mock_logging_obj - ): + def test_sync_image_generation_without_headers(self, openai_chat_completions, mock_logging_obj): """Sync image_generation without headers should not inject extra_headers.""" mock_image_data = MagicMock() mock_image_data.model_dump.return_value = { @@ -90,9 +85,7 @@ class TestImageGenerationExtraHeaders: assert "extra_headers" not in kwargs @pytest.mark.asyncio - async def test_async_image_generation_with_headers( - self, openai_chat_completions, mock_logging_obj - ): + async def test_async_image_generation_with_headers(self, openai_chat_completions, mock_logging_obj): """Async aimage_generation should pass headers as extra_headers to images.generate().""" mock_image_data = MagicMock() mock_image_data.model_dump.return_value = { @@ -121,9 +114,7 @@ class TestImageGenerationExtraHeaders: assert kwargs.get("extra_headers") == test_headers @pytest.mark.asyncio - async def test_async_image_generation_without_headers( - self, openai_chat_completions, mock_logging_obj - ): + async def test_async_image_generation_without_headers(self, openai_chat_completions, mock_logging_obj): """Async aimage_generation without headers should not inject extra_headers.""" mock_image_data = MagicMock() mock_image_data.model_dump.return_value = { @@ -193,20 +184,14 @@ class TestImageGenerationExtraHeaders: client=mock_openai_client, ) - logged_body = mock_logging_obj.pre_call.call_args[1]["additional_args"][ - "complete_input_dict" - ] + logged_body = mock_logging_obj.pre_call.call_args[1]["additional_args"]["complete_input_dict"] assert "extra_headers" not in logged_body _, kwargs = mock_openai_client.images.generate.call_args assert kwargs.get("extra_headers") == test_headers - def test_sync_image_generation_forwards_headers_to_async( - self, openai_chat_completions, mock_logging_obj - ): + def test_sync_image_generation_forwards_headers_to_async(self, openai_chat_completions, mock_logging_obj): """When aimg_generation=True, image_generation should forward headers to aimage_generation.""" - with patch.object( - openai_chat_completions, "aimage_generation" - ) as mock_aimage_gen: + with patch.object(openai_chat_completions, "aimage_generation") as mock_aimage_gen: mock_aimage_gen.return_value = MagicMock() test_headers = {"x-custom-header": "value"} @@ -226,6 +211,97 @@ class TestImageGenerationExtraHeaders: call_kwargs = mock_aimage_gen.call_args[1] assert call_kwargs["headers"] == test_headers + @pytest.mark.parametrize("is_async", [False, True]) + @pytest.mark.asyncio + async def test_optional_params_extra_headers_never_enter_the_json_body( + self, openai_chat_completions, mock_logging_obj, is_async + ): + """Proxy credentials put extra_headers on optional_params (and often + extra_body). Those must reach images.generate only as the SDK + extra_headers kwarg. OpenAI rejects extra_headers in the JSON body.""" + mock_image_data = MagicMock() + mock_image_data.model_dump.return_value = { + "created": 1700000000, + "data": [{"url": "https://example.com/image.png"}], + } + + mock_openai_client = MagicMock() + mock_openai_client.api_key = "test-key" + mock_openai_client._base_url._uri_reference = "https://api.openai.com" + + test_headers = {"cf-aig-authorization": "Bearer custom-token"} + optional_params = { + "size": "1024x1024", + "quality": "low", + "extra_headers": test_headers, + "extra_body": {"extra_headers": test_headers, "foo": "bar"}, + } + + if is_async: + mock_openai_client.images.generate = AsyncMock(return_value=mock_image_data) + await openai_chat_completions.aimage_generation( + prompt="A white cat", + data={"model": "gpt-image-2", "prompt": "A white cat", **optional_params}, + model_response=MagicMock(), + timeout=60.0, + logging_obj=mock_logging_obj, + api_key="test-key", + client=mock_openai_client, + ) + else: + mock_openai_client.images.generate.return_value = mock_image_data + openai_chat_completions.image_generation( + model="gpt-image-2", + prompt="A white cat", + timeout=60.0, + optional_params=optional_params, + logging_obj=mock_logging_obj, + api_key="test-key", + client=mock_openai_client, + ) + + _, kwargs = mock_openai_client.images.generate.call_args + assert kwargs.get("extra_headers") == test_headers + assert kwargs.get("size") == "1024x1024" + assert kwargs.get("quality") == "low" + extra_body = kwargs.get("extra_body") or {} + assert "extra_headers" not in extra_body + assert extra_body.get("foo") == "bar" + + logged_body = mock_logging_obj.pre_call.call_args[1]["additional_args"]["complete_input_dict"] + assert "extra_headers" not in logged_body + assert "extra_headers" not in (logged_body.get("extra_body") or {}) + + def test_optional_params_headers_merge_prefers_caller_headers(self, openai_chat_completions, mock_logging_obj): + """When both optional_params and the headers arg carry extra_headers, + the explicit headers arg wins, matching the previous overwrite.""" + mock_image_data = MagicMock() + mock_image_data.model_dump.return_value = { + "created": 1700000000, + "data": [{"url": "https://example.com/image.png"}], + } + + mock_openai_client = MagicMock() + mock_openai_client.images.generate.return_value = mock_image_data + mock_openai_client.api_key = "test-key" + mock_openai_client._base_url._uri_reference = "https://api.openai.com" + + openai_chat_completions.image_generation( + model="gpt-image-2", + prompt="A white cat", + timeout=60.0, + optional_params={ + "extra_headers": {"cf-aig-authorization": "Bearer from-optional"}, + }, + logging_obj=mock_logging_obj, + api_key="test-key", + headers={"cf-aig-authorization": "Bearer from-headers"}, + client=mock_openai_client, + ) + + _, kwargs = mock_openai_client.images.generate.call_args + assert kwargs.get("extra_headers") == {"cf-aig-authorization": "Bearer from-headers"} + class TestImageGenerationEntryPointHeaders: """Test that litellm.image_generation() passes headers through to the OpenAI provider.""" @@ -259,3 +335,4 @@ class TestImageGenerationEntryPointHeaders: mock_openai_client.images.generate.assert_called_once() _, kwargs = mock_openai_client.images.generate.call_args assert kwargs.get("extra_headers") == test_headers + assert "extra_headers" not in (kwargs.get("extra_body") or {}) From 313e1495fa2156e03ab50be784a0ed806d3e7e42 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 16 Sep 2026 01:31:27 +0000 Subject: [PATCH 7/8] fix(openai): sanitize image generation failure telemetry Exception post_call paths were logging the raw request dict, which could leak extra_headers and nested bearer tokens. Use the sanitized logged body on every failure path and keep extra_body/extra_query keys that are not headers. Co-authored-by: Zsanz3 --- litellm/llms/openai/openai.py | 27 ++++--- ...t_openai_image_generation_extra_headers.py | 78 ++++++++++++++++++- 2 files changed, 92 insertions(+), 13 deletions(-) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 0c6cff5bca5..69ff2e9b8f9 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -362,13 +362,22 @@ def _embedding_request_without_sdk_defaults( return body, options +def _image_generation_logged_body(data: Mapping[str, object]) -> dict[str, object]: + extra_body: Final = { + key: item for key, item in _as_mapping(data.get("extra_body")).items() if key != "extra_headers" + } + extra_query: Final = dict(_as_mapping(data.get("extra_query"))) + return { # mutable-ok: loggers isinstance-check this payload as a dict + **{key: item for key, item in data.items() if key not in _SDK_OPTION_KEYS}, + **({} if not extra_query else {"extra_query": extra_query}), + **({} if not extra_body else {"extra_body": extra_body}), + } + + def _image_generation_request_data( data: Mapping[str, object], headers: Mapping[str, object] | None ) -> tuple[dict[str, object], Mapping[str, object]]: extra_body_source: Final = _as_mapping(data.get("extra_body")) - extra_body: Final = types.MappingProxyType( - {key: item for key, item in extra_body_source.items() if key != "extra_headers"} - ) extra_headers: Final = types.MappingProxyType( { **_as_mapping(extra_body_source.get("extra_headers")), @@ -376,17 +385,12 @@ def _image_generation_request_data( **_as_mapping(headers), } ) - extra_query: Final = _as_mapping(data.get("extra_query")) - logged_body: Final = { # mutable-ok: loggers isinstance-check this payload as a dict - key: item for key, item in data.items() if key not in _SDK_OPTION_KEYS - } + logged_body: Final = _image_generation_logged_body(data) request_data: Final = types.MappingProxyType( dict( ( *logged_body.items(), *((("extra_headers", extra_headers),) if extra_headers else ()), - *((("extra_query", extra_query),) if extra_query else ()), - *((("extra_body", extra_body),) if extra_body else ()), ) ) ) @@ -1483,6 +1487,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): logging_obj.post_call( input=prompt, api_key=api_key, + additional_args={"complete_input_dict": _image_generation_logged_body(data)}, original_response=str(e), ) raise e @@ -1569,7 +1574,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): logging_obj.post_call( input=prompt, api_key=api_key, - additional_args={"complete_input_dict": data}, + additional_args={"complete_input_dict": _image_generation_logged_body(data)}, original_response=str(e), ) raise e @@ -1578,7 +1583,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): logging_obj.post_call( input=prompt, api_key=api_key, - additional_args={"complete_input_dict": data}, + additional_args={"complete_input_dict": _image_generation_logged_body(data)}, original_response=str(e), ) if hasattr(e, "status_code"): diff --git a/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py b/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py index b422e62b9c2..c7378a447a0 100644 --- a/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py +++ b/tests/test_litellm/llms/openai/image_generation/test_openai_image_generation_extra_headers.py @@ -10,8 +10,24 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from litellm.llms.openai.common_utils import OpenAIError from litellm.llms.openai.openai import OpenAIChatCompletion +_SECRET_HEADERS = {"Authorization": "Bearer leaked-token", "cf-aig-authorization": "Bearer custom-token"} + + +def _assert_logged_body_omits_credentials(logged_body: object) -> None: + assert isinstance(logged_body, dict) + assert "extra_headers" not in logged_body + assert "Authorization" not in logged_body + assert "cf-aig-authorization" not in logged_body + extra_body = logged_body.get("extra_body") or {} + assert "extra_headers" not in extra_body + assert "Bearer leaked-token" not in str(logged_body) + assert extra_body.get("foo") == "bar" + assert logged_body.get("extra_query") == {"user": "alice"} + assert logged_body.get("size") == "1024x1024" + @pytest.fixture def openai_chat_completions(): @@ -235,6 +251,7 @@ class TestImageGenerationExtraHeaders: "quality": "low", "extra_headers": test_headers, "extra_body": {"extra_headers": test_headers, "foo": "bar"}, + "extra_query": {"user": "alice"}, } if is_async: @@ -267,10 +284,10 @@ class TestImageGenerationExtraHeaders: extra_body = kwargs.get("extra_body") or {} assert "extra_headers" not in extra_body assert extra_body.get("foo") == "bar" + assert kwargs.get("extra_query") == {"user": "alice"} logged_body = mock_logging_obj.pre_call.call_args[1]["additional_args"]["complete_input_dict"] - assert "extra_headers" not in logged_body - assert "extra_headers" not in (logged_body.get("extra_body") or {}) + _assert_logged_body_omits_credentials(logged_body) def test_optional_params_headers_merge_prefers_caller_headers(self, openai_chat_completions, mock_logging_obj): """When both optional_params and the headers arg carry extra_headers, @@ -302,6 +319,63 @@ class TestImageGenerationExtraHeaders: _, kwargs = mock_openai_client.images.generate.call_args assert kwargs.get("extra_headers") == {"cf-aig-authorization": "Bearer from-headers"} + @pytest.mark.parametrize( + "is_async,raised", + [ + (False, OpenAIError(status_code=400, message="bad image params")), + (False, RuntimeError("forced failure")), + (True, OpenAIError(status_code=400, message="bad image params")), + (True, RuntimeError("forced failure")), + ], + ids=["sync-openai-error", "sync-runtime-error", "async-openai-error", "async-runtime-error"], + ) + @pytest.mark.asyncio + async def test_failure_telemetry_omits_extra_headers( + self, openai_chat_completions, mock_logging_obj, is_async, raised + ): + """post_call on a generate() failure must use the sanitized body, not raw data.""" + mock_openai_client = MagicMock() + mock_openai_client.api_key = "test-key" + mock_openai_client._base_url._uri_reference = "https://api.openai.com" + + optional_params = { + "size": "1024x1024", + "extra_headers": _SECRET_HEADERS, + "extra_body": {"extra_headers": _SECRET_HEADERS, "foo": "bar"}, + "extra_query": {"user": "alice"}, + } + + if is_async: + mock_openai_client.images.generate = AsyncMock(side_effect=raised) + with pytest.raises(type(raised)): + await openai_chat_completions.aimage_generation( + prompt="A white cat", + data={"model": "gpt-image-2", "prompt": "A white cat", **optional_params}, + model_response=MagicMock(), + timeout=60.0, + logging_obj=mock_logging_obj, + api_key="test-key", + headers=_SECRET_HEADERS, + client=mock_openai_client, + ) + else: + mock_openai_client.images.generate.side_effect = raised + with pytest.raises(OpenAIError): + openai_chat_completions.image_generation( + model="gpt-image-2", + prompt="A white cat", + timeout=60.0, + optional_params=optional_params, + logging_obj=mock_logging_obj, + api_key="test-key", + headers=_SECRET_HEADERS, + client=mock_openai_client, + ) + + mock_openai_client.images.generate.assert_called_once() + logged_body = mock_logging_obj.post_call.call_args[1]["additional_args"]["complete_input_dict"] + _assert_logged_body_omits_credentials(logged_body) + class TestImageGenerationEntryPointHeaders: """Test that litellm.image_generation() passes headers through to the OpenAI provider.""" From a3da59f2bed92a24f6d4ec60a34ce3fe25995b72 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 26 Oct 2026 13:37:12 +0000 Subject: [PATCH 8/8] test(openai): point image extra_headers coverage at with_raw_response Main now reads provider headers off the raw response, so the proxy credential cases mock that call and pass a real ImageResponse. Co-authored-by: Zsanz3 --- .../test_openai_image_generation_extra_headers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/llms/openai/image_generation/test_openai_image_generation_extra_headers.py b/tests/unit/llms/openai/image_generation/test_openai_image_generation_extra_headers.py index d1a9297267f..a0195ac3484 100644 --- a/tests/unit/llms/openai/image_generation/test_openai_image_generation_extra_headers.py +++ b/tests/unit/llms/openai/image_generation/test_openai_image_generation_extra_headers.py @@ -275,7 +275,7 @@ class TestImageGenerationExtraHeaders: await openai_chat_completions.aimage_generation( prompt="A white cat", data={"model": "gpt-image-2", "prompt": "A white cat", **optional_params}, - model_response=MagicMock(), + model_response=ImageResponse(), timeout=60.0, logging_obj=mock_logging_obj, api_key="test-key",